mirror of
https://github.com/dadosfera/maestro.git
synced 2026-08-31 19:58:21 +00:00
Compare commits
40
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8eb7fd0169 | ||
|
|
8a9d6c2f9c | ||
|
|
2e181e70af | ||
|
|
ef615adb8e | ||
|
|
e03900811b | ||
|
|
5bc5fb0977 | ||
|
|
011032e3d4 | ||
|
|
17363e74f4 | ||
|
|
63efff6adf | ||
|
|
b5f569e522 | ||
|
|
5c77577992 | ||
|
|
4b9e113185 | ||
|
|
06d505c50a | ||
|
|
c70abfc826 | ||
|
|
db2c7d6c02 | ||
|
|
0a99ce1aa4 | ||
|
|
39c66f030a | ||
|
|
1223e21ac4 | ||
|
|
2177f6725c | ||
|
|
65ab16236f | ||
|
|
b4cc8151d7 | ||
|
|
e8b982998f | ||
|
|
ccd4159c59 | ||
|
|
9e49abb40d | ||
|
|
21a82b64f3 | ||
|
|
2414fcf21e | ||
|
|
94fbdb2226 | ||
|
|
985170d7ae | ||
|
|
b38b8f51e2 | ||
|
|
ea16e62d6b | ||
|
|
6182705410 | ||
|
|
a5d78a97ae | ||
|
|
2b33c22149 | ||
|
|
e32787baff | ||
|
|
f449d8ebd9 | ||
|
|
02627d023a | ||
|
|
cf94c73648 | ||
|
|
6b3241281b | ||
|
|
fe1caa003e | ||
|
|
e980c58507 |
@@ -71,6 +71,11 @@ jobs:
|
||||
sudo mv helmfile /usr/local/bin/
|
||||
helmfile --version
|
||||
|
||||
- name: Install Helm Diff plugin
|
||||
run: |
|
||||
helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
|
||||
helm diff version
|
||||
|
||||
- name: Debug Helm env
|
||||
run: |
|
||||
helm env
|
||||
@@ -102,4 +107,5 @@ jobs:
|
||||
- name: Run Helmfile Diff
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
HELM_PLUGINS: /home/runner/.local/share/helm/plugins
|
||||
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
FROM node:20-alpine AS base_image
|
||||
RUN npm install -g npm@latest
|
||||
RUN npm install -g npm@10.8.2
|
||||
|
||||
FROM base_image AS build_base
|
||||
WORKDIR /app
|
||||
|
||||
@@ -113,6 +113,8 @@ spec:
|
||||
value: {{ .Values.maestro.platform_api_url }}
|
||||
- name: STORAGE_EXPLORER_API_URL
|
||||
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
|
||||
- name: FIREBASE_BASE_URL
|
||||
value: {{ .Values.maestro.firebase_base_url }}
|
||||
- name: JWT_PRIVATE_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -10,6 +10,7 @@ maestro:
|
||||
redis_database: "1"
|
||||
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
|
||||
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
|
||||
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
|
||||
|
||||
hostname: maestro.stg.dadosfera.ai
|
||||
|
||||
|
||||
@@ -55,6 +55,7 @@ maestro:
|
||||
redis_database: "0"
|
||||
redis_tls: "true"
|
||||
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
|
||||
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
|
||||
+297
-90
@@ -3343,14 +3343,7 @@
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
@@ -3541,6 +3534,64 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/pipelinesV2/{id}/data-assets": {
|
||||
"get": {
|
||||
"operationId": "PipelinesController_findAllDataAssetByPipeline",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "id",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "object",
|
||||
"required": true,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"PipelinesV2"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/pipelinesV2/{pipelineId}/inputs/{id}": {
|
||||
"patch": {
|
||||
"operationId": "PipelinesController_updatePipelineInput",
|
||||
@@ -3589,6 +3640,46 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/pipelinesV2/{id}/upgrade": {
|
||||
"patch": {
|
||||
"operationId": "PipelinesController_upgradeConnector",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "id",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"204": {
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"PipelinesV2"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/pipelinesV2/init-upload": {
|
||||
"post": {
|
||||
"operationId": "PipelinesController_initUploadFile",
|
||||
@@ -3776,88 +3867,6 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/pipelines/start/{id}": {
|
||||
"post": {
|
||||
"operationId": "PipelinesController_activate",
|
||||
"summary": "",
|
||||
"deprecated": true,
|
||||
"description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "id",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Pipelines"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
},
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/pipelines/{id}/status": {
|
||||
"get": {
|
||||
"operationId": "PipelinesController_getPipelineStatus",
|
||||
"summary": "",
|
||||
"deprecated": true,
|
||||
"description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "id",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Pipelines"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
},
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/transformations": {
|
||||
"post": {
|
||||
"operationId": "TransformationsController_create",
|
||||
@@ -4561,6 +4570,62 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": {
|
||||
"get": {
|
||||
"operationId": "PlatformApiController_getPipelineRunJobs",
|
||||
"summary": "Get pipeline run jobs",
|
||||
"description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "pipelineId",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "runId",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "DB-backed job runs for the selected pipeline run.",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"jobs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"jobs"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Platform API"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/jobs/{jobId}/input": {
|
||||
"put": {
|
||||
"operationId": "PlatformApiController_updateJobInput",
|
||||
@@ -5431,6 +5496,48 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/catalog/schemas": {
|
||||
"get": {
|
||||
"operationId": "CatalogController_findSchemas",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Catalog"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
},
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/catalog/data-asset/{id}": {
|
||||
"get": {
|
||||
"operationId": "CatalogController_getDataAsset",
|
||||
@@ -5845,6 +5952,66 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/catalog/data-asset/{id}/certification-status": {
|
||||
"put": {
|
||||
"operationId": "CatalogController_updateDataAssetCertificationStatus",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "id",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/IUpdateCertificationStatusRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Catalog"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
},
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/catalog/data-asset/{id}/manage-permissions": {
|
||||
"put": {
|
||||
"operationId": "CatalogController_manageDataAssetPermissions",
|
||||
@@ -8496,6 +8663,29 @@
|
||||
"Health"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/release_note": {
|
||||
"get": {
|
||||
"operationId": "ReleaseNoteController_getLatestReleaseNote",
|
||||
"parameters": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"info": {
|
||||
@@ -10902,6 +11092,23 @@
|
||||
"data_asset"
|
||||
]
|
||||
},
|
||||
"IUpdateCertificationStatusRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"certification_status": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"draft",
|
||||
"in_review",
|
||||
"approved",
|
||||
"deprecated"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"certification_status"
|
||||
]
|
||||
},
|
||||
"ICreateDataAsset": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
Generated
+4
-14
@@ -16,8 +16,7 @@
|
||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.40.0-beta.5",
|
||||
"@dadosfera/protospack-v2": "^3.40.0-beta.10",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -1735,19 +1734,10 @@
|
||||
"winston-log2gelf": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack": {
|
||||
"version": "2.5.3",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
|
||||
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"rxjs": "^7.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.40.0-beta.5",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.5.tgz",
|
||||
"integrity": "sha512-iocKv/XXp2jKAasO5ONgm31cKfLgNsU4pEKZMr6YnR7nQaH11WcW7rnuagNxWoik++wLUqbYyf0bZWRDzMlCPA==",
|
||||
"version": "3.40.0-beta.10",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
|
||||
"integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"rxjs": "^7.5.5"
|
||||
|
||||
+1
-2
@@ -34,8 +34,7 @@
|
||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.40.0-beta.5",
|
||||
"@dadosfera/protospack-v2": "^3.40.0-beta.10",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
|
||||
+3
-2
@@ -17,7 +17,6 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
|
||||
import { NetworkConfigModule } from './modules/network-config/network-config.module';
|
||||
import { InputsModule } from './modules/inputs/inputs.module';
|
||||
import { OauthModule } from './modules/oauth/oauth.module';
|
||||
import { PipelinesModule } from './modules/pipelines/pipelines.module';
|
||||
import { TransformationsModule } from './modules/transformations/transformations.module';
|
||||
import { HealthModule } from './modules/health/health.module';
|
||||
import { CatalogModule } from './modules/catalog/catalog.module';
|
||||
@@ -35,6 +34,8 @@ import { ShareMetadataModule } from './modules/share-metadata/share-metadata.mod
|
||||
import { ApiKeyModule } from './modules/api-key/api-key.module';
|
||||
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
|
||||
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
|
||||
import { ReleaseNoteModule } from './modules/release_note/release_note.module';
|
||||
|
||||
|
||||
@Module({
|
||||
providers: [
|
||||
@@ -58,7 +59,6 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor
|
||||
PermissionsModule,
|
||||
TermsOfUseModule,
|
||||
ConnectionTestModule,
|
||||
PipelinesModule,
|
||||
TransformationsModule,
|
||||
UsersModule,
|
||||
RolesModule,
|
||||
@@ -79,6 +79,7 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor
|
||||
StorageExplorerModule,
|
||||
//Always leave HealthModule last, so it is on the bottom of swagger
|
||||
HealthModule,
|
||||
ReleaseNoteModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -153,6 +153,7 @@ export class AuthenticationGuard
|
||||
user_id: accessTokenPayload.user_id,
|
||||
username: accessTokenPayload.username,
|
||||
permissions: accessTokenPayload.permissions,
|
||||
roles: accessTokenPayload.roles,
|
||||
customer_id: accessTokenPayload.customer_id,
|
||||
customer_name: accessTokenPayload.customer_name,
|
||||
customer_tier: accessTokenPayload.customer_tier,
|
||||
|
||||
@@ -11,6 +11,7 @@ export function extractUserFrom(aRawJwt: string) {
|
||||
user_id: payload.user_id,
|
||||
username: payload.username,
|
||||
permissions: payload.permissions,
|
||||
roles: payload.roles,
|
||||
customer_id: payload.customer_id,
|
||||
customer_name: payload.customer_name,
|
||||
customer_tier: payload.customer_tier,
|
||||
|
||||
@@ -357,6 +357,16 @@ export const PERMISSIONS_GROUPS = {
|
||||
'es-es': 'Crear y editar atributos en el catálogo',
|
||||
},
|
||||
},
|
||||
CERTIFY: {
|
||||
seqid: 53,
|
||||
claim: 'catalog:certify',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Alterar o status de certificação dos Ativos',
|
||||
'en-us': "Change Assets' certification status",
|
||||
'es-es': 'Cambiar el estado de certificación de los Activos',
|
||||
},
|
||||
},
|
||||
DELETE: {
|
||||
seqid: 1,
|
||||
claim: 'catalog:delete',
|
||||
|
||||
@@ -12,6 +12,7 @@ export interface RequestUser {
|
||||
customer_tier: string;
|
||||
access_token: string;
|
||||
customer_modules: string[];
|
||||
roles: string[];
|
||||
}
|
||||
|
||||
export const User: (options?: { required?: boolean }) => ParameterDecorator =
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
HttpStatus,
|
||||
Res,
|
||||
} from '@nestjs/common';
|
||||
import { ValidationPipe } from '../../pipes/object-validation.pipe';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
ApiHeaders,
|
||||
@@ -46,6 +47,7 @@ import {
|
||||
IMakeAComment,
|
||||
IOneDataAsset,
|
||||
IPreviewResponse,
|
||||
IUpdateCertificationStatusRequest,
|
||||
IUpdateDataRequest,
|
||||
TriggerCatalogReq,
|
||||
TriggerCatalogRes,
|
||||
@@ -241,6 +243,34 @@ export class CatalogController {
|
||||
return res;
|
||||
}
|
||||
|
||||
@Get('schemas')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async findSchemas(@User() user: RequestUser) {
|
||||
const { username, user_id, customer_id, customer_name } = user;
|
||||
this.logger.info(`/catalog - ON FIND SCHEMAS ROUTE`, {
|
||||
username,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
username,
|
||||
user_id,
|
||||
customer_id,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
try {
|
||||
const res = await this.catalogService.findSchemas(metadata);
|
||||
return res;
|
||||
} catch (error) {
|
||||
throw new HttpException(error.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Get('data-asset/:id')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
@@ -465,6 +495,8 @@ export class CatalogController {
|
||||
language,
|
||||
});
|
||||
|
||||
delete (body as any).certification_status;
|
||||
|
||||
const result = await this.catalogService.updateOneDataAsset({
|
||||
body,
|
||||
data_asset_id,
|
||||
@@ -478,6 +510,33 @@ export class CatalogController {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Put('data-asset/:id/certification-status')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async updateDataAssetCertificationStatus(
|
||||
@User() user: RequestUser,
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') data_asset_id: string,
|
||||
@Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
|
||||
): Promise<IUpdateCertificationStatusRequest> {
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
language,
|
||||
});
|
||||
|
||||
return this.catalogService.updateCertificationStatus({
|
||||
body,
|
||||
data_asset_id,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
@Post('data-asset/:id/docs')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||
|
||||
@@ -5,20 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { CatalogController } from './catalog.controller';
|
||||
import { CatalogClientConfiguration } from './catalog-client';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||
import { UsersModule } from '../users/users.module';
|
||||
import { RolesModule } from '../roles/roles.module';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
import { ShareModule } from './share/share.module';
|
||||
import { CatalogService } from './catalog.service';
|
||||
import { MixpanelModule } from '../mixpanel/mixpanel.module';
|
||||
|
||||
const client = new CatalogClientConfiguration();
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ClientsModule.register([client.providerOptions]),
|
||||
OldPipelineModule,
|
||||
UsersModule,
|
||||
RolesModule,
|
||||
CustomersModule,
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
AssetReporter,
|
||||
BatchRemoveRlsRulesRequest,
|
||||
CreateDataDocsDTO,
|
||||
IUpdateCertificationStatusRequest,
|
||||
IUpdateDataRequest,
|
||||
TriggerCatalogReq,
|
||||
} from './dtos';
|
||||
@@ -214,15 +215,6 @@ class CatalogService implements OnModuleInit {
|
||||
|
||||
this.logger.debug('Extracted filters:', { filters });
|
||||
|
||||
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
|
||||
search,
|
||||
page,
|
||||
size,
|
||||
sort_by,
|
||||
order,
|
||||
filters,
|
||||
});
|
||||
|
||||
if (
|
||||
filters.manually !== undefined &&
|
||||
filters.manually !== null &&
|
||||
@@ -233,15 +225,6 @@ class CatalogService implements OnModuleInit {
|
||||
delete filters.manually;
|
||||
}
|
||||
|
||||
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
|
||||
search,
|
||||
page,
|
||||
size,
|
||||
sort_by,
|
||||
order,
|
||||
filters,
|
||||
});
|
||||
|
||||
if (filters.owner) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
@@ -402,6 +385,28 @@ class CatalogService implements OnModuleInit {
|
||||
return { data_asset: asset[0] };
|
||||
}
|
||||
|
||||
async updateCertificationStatus(data: {
|
||||
data_asset_id: string;
|
||||
body: IUpdateCertificationStatusRequest;
|
||||
metadata: Metadata;
|
||||
}) {
|
||||
const { body, data_asset_id, metadata } = data;
|
||||
|
||||
await lastValueFrom(
|
||||
this.catalogWriteService.UpdateDataAsset(
|
||||
{
|
||||
id: data_asset_id,
|
||||
changes: JSON.stringify({
|
||||
certification_status: body.certification_status,
|
||||
}),
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
|
||||
return { certification_status: body.certification_status };
|
||||
}
|
||||
|
||||
async updateOneDataAsset(data: {
|
||||
data_asset_id: string;
|
||||
customer_id: string;
|
||||
@@ -517,6 +522,22 @@ class CatalogService implements OnModuleInit {
|
||||
return response;
|
||||
}
|
||||
|
||||
async findSchemas(metadata: Metadata) {
|
||||
this.logger.info('CatalogService - findSchemas');
|
||||
|
||||
try {
|
||||
const response = await lastValueFrom(
|
||||
this.catalogReadService.GetSchemas({}, metadata),
|
||||
);
|
||||
|
||||
return response;
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error('Error fetching schemas:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
|
||||
import { IsEnum } from 'class-validator';
|
||||
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||
|
||||
export enum DataAssetShareType {
|
||||
@@ -6,6 +7,12 @@ export enum DataAssetShareType {
|
||||
public = 'public',
|
||||
private = 'private',
|
||||
}
|
||||
export enum CertificationStatus {
|
||||
draft = 'draft',
|
||||
in_review = 'in_review',
|
||||
approved = 'approved',
|
||||
deprecated = 'deprecated',
|
||||
}
|
||||
export enum OrderEnum {
|
||||
asc = 'asc',
|
||||
desc = 'desc',
|
||||
@@ -205,6 +212,13 @@ export class IUpdateDataRequest {
|
||||
@ApiPropertyOptional()
|
||||
docs?: string;
|
||||
}
|
||||
|
||||
export class IUpdateCertificationStatusRequest {
|
||||
@ApiProperty({ enum: CertificationStatus })
|
||||
@IsEnum(CertificationStatus)
|
||||
certification_status: CertificationStatus;
|
||||
}
|
||||
|
||||
export class ICreateDataAsset implements CreateDataAssetRequest {
|
||||
@ApiProperty()
|
||||
display_name: string;
|
||||
|
||||
@@ -262,6 +262,7 @@ export class ShareService implements OnModuleInit {
|
||||
user_id: accessTokenPayload.user_id,
|
||||
username: accessTokenPayload.username,
|
||||
permissions: accessTokenPayload.permissions,
|
||||
roles: accessTokenPayload.roles,
|
||||
customer_id: accessTokenPayload.customer_id,
|
||||
customer_name: accessTokenPayload.customer_name,
|
||||
customer_tier: accessTokenPayload.customer_tier,
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
export interface Info {
|
||||
user_id: string;
|
||||
customer_id: string;
|
||||
customer: string;
|
||||
}
|
||||
|
||||
interface Values {
|
||||
jdbc_user: string;
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import {
|
||||
PipelineServicesNames,
|
||||
PipelinesServiceInterface,
|
||||
} from '@dadosfera/protospack';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
|
||||
import { IIdRequest } from './interfaces';
|
||||
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
|
||||
export class PipelinesClientService implements OnModuleInit {
|
||||
private pipelineService: PipelinesServiceInterface;
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
@Inject(PipelinesClientConfiguration.name)
|
||||
private readonly grpcClient: ClientGrpc,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
this.pipelineService =
|
||||
this.grpcClient.getService<PipelinesServiceInterface>(
|
||||
PipelineServicesNames.PipelineService,
|
||||
);
|
||||
}
|
||||
|
||||
async getPipelineStatus(data) {
|
||||
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
||||
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineService.getPipelineStatus(data),
|
||||
)
|
||||
.then((res) => {
|
||||
const statusArray =
|
||||
res.status?.sort((a, b) => {
|
||||
if (a.id < b.id) {
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}) || [];
|
||||
return { status: statusArray };
|
||||
})
|
||||
.catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
this.logger.info('Done');
|
||||
|
||||
return statusPipelineResponse;
|
||||
}
|
||||
|
||||
async runPipeline({ id, info }: IIdRequest) {
|
||||
this.logger.info('PipelinesClientService - RunPipeline');
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineService.triggerPipeline({ id, info }),
|
||||
).catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
|
||||
if (statusPipelineResponse.status == false) {
|
||||
throw new ConflictException(
|
||||
'This pipeline is not ready yet to execute, Try again later!',
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.info('Done');
|
||||
return statusPipelineResponse;
|
||||
}
|
||||
}
|
||||
-36
@@ -1,36 +0,0 @@
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export interface ICreatePipelineDto {
|
||||
input: IdRequest;
|
||||
transformations: IdRequest[];
|
||||
output: IdRequest;
|
||||
tags: string[];
|
||||
name: string;
|
||||
description: string;
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface IdRequest {
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface IIdRequest {
|
||||
id: string;
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface IUpdatePipelineRequest {
|
||||
input: IdRequest;
|
||||
transformations: IdRequest[];
|
||||
output: IdRequest;
|
||||
tags: string[];
|
||||
name: string;
|
||||
description: string;
|
||||
id: string;
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface IGetPipelineLogsRequest {
|
||||
id: string;
|
||||
details: string;
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
import {
|
||||
ClientsProviderAsyncOptions,
|
||||
GrpcOptions,
|
||||
Transport,
|
||||
} from '@nestjs/microservices';
|
||||
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
|
||||
import { credentials } from '@grpc/grpc-js';
|
||||
|
||||
const isLocalConnection =
|
||||
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
||||
process.env.PIFACTORY_URL.includes('0.0.0.0');
|
||||
|
||||
export class PipelinesClientConfiguration {
|
||||
public name = 'PipelinesClientConfiguration';
|
||||
private config: GrpcOptions = {
|
||||
transport: Transport.GRPC,
|
||||
options: {
|
||||
url: process.env.PIFACTORY_URL,
|
||||
package: PipelinePackages,
|
||||
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
||||
protoPath: PipelineProtoFilePath,
|
||||
loader: {
|
||||
keepCase: true,
|
||||
enums: String,
|
||||
defaults: false,
|
||||
},
|
||||
},
|
||||
};
|
||||
providerOptions: ClientsProviderAsyncOptions = {
|
||||
name: this.name,
|
||||
...this.config,
|
||||
};
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
|
||||
import { ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||
import {
|
||||
AuthenticateCondition,
|
||||
Authenticated,
|
||||
RequireSomePermission,
|
||||
} from 'src/decorators/authentication.decorator';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { PipelinesService } from './pipelines.service';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||
|
||||
@ApiInternalOnlyController()
|
||||
@ApiTags('Pipelines')
|
||||
@Controller('pipelines')
|
||||
@Authenticated()
|
||||
export class PipelinesController {
|
||||
logger: DadosferaLogger;
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private pipelineService: PipelinesService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Post('start/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
@ApiOperation({
|
||||
deprecated: true,
|
||||
description:
|
||||
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
|
||||
})
|
||||
async activate(@Param('id') id: string, @Body() body) {
|
||||
const { info } = body;
|
||||
|
||||
this.logger.info(
|
||||
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
|
||||
{
|
||||
user: body.info.user_id,
|
||||
customer: body.info.customer,
|
||||
},
|
||||
);
|
||||
|
||||
const response = await this.pipelineService.runPipeline({ id, info });
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
@Get(':id/status')
|
||||
@ApiOperation({
|
||||
deprecated: true,
|
||||
description:
|
||||
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
|
||||
})
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||
body.id = id;
|
||||
|
||||
this.logger.info(
|
||||
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
|
||||
{
|
||||
user: body.info.user_id,
|
||||
customer: body.info.customer,
|
||||
},
|
||||
);
|
||||
|
||||
const response = await this.pipelineService.getPipelineStatus(body);
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { PipelinesController } from './pipelines.controller';
|
||||
import { PipelinesService } from './pipelines.service';
|
||||
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
import { PipelinesClientService } from './client.service';
|
||||
|
||||
const client = new PipelinesClientConfiguration();
|
||||
|
||||
@Module({
|
||||
imports: [ClientsModule.register([client.providerOptions])],
|
||||
controllers: [PipelinesController],
|
||||
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
|
||||
exports: [PipelinesService],
|
||||
})
|
||||
export class PipelinesModule {}
|
||||
@@ -1,33 +0,0 @@
|
||||
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
|
||||
import { PipelinesClientService } from './client.service';
|
||||
import { IIdRequest } from './interfaces';
|
||||
import { objectCamelToSnake } from 'src/utils/CaseConverter';
|
||||
|
||||
@Injectable()
|
||||
export class PipelinesService {
|
||||
constructor(private pipelineClient: PipelinesClientService) {}
|
||||
|
||||
async getPipelineStatus(data: IIdRequest) {
|
||||
try {
|
||||
const pipelineStatusResponse =
|
||||
await this.pipelineClient.getPipelineStatus(data);
|
||||
|
||||
return objectCamelToSnake(pipelineStatusResponse);
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
async runPipeline({ id, info }: IIdRequest) {
|
||||
try {
|
||||
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
|
||||
id,
|
||||
info,
|
||||
});
|
||||
|
||||
return objectCamelToSnake(triggerPipelineResponse);
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export class PipelineInputsDTO {
|
||||
@ApiProperty()
|
||||
@@ -62,6 +61,12 @@ export interface IIdRequest {
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface Info {
|
||||
user_id: string;
|
||||
customer_id: string;
|
||||
customer: string;
|
||||
}
|
||||
|
||||
export interface IUpdatePipelineRequest {
|
||||
input: IdRequest;
|
||||
transformations: IdRequest[];
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
HttpException,
|
||||
BadRequestException,
|
||||
UseGuards,
|
||||
Res,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
@@ -34,7 +35,6 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
|
||||
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
|
||||
import {
|
||||
ICompleteUploadCSVFile,
|
||||
ICreatePipelineCSVFile,
|
||||
@@ -63,9 +63,7 @@ export class PipelinesController {
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
|
||||
private pipelinesClientService: PipelinesService,
|
||||
private oldPipelinesService: OldPipelineService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
@@ -193,6 +191,7 @@ export class PipelinesController {
|
||||
@Get(':id/status')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||
|
||||
body.id = id;
|
||||
|
||||
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
|
||||
@@ -200,7 +199,7 @@ export class PipelinesController {
|
||||
customer: body.info.customer,
|
||||
});
|
||||
|
||||
const response = await this.oldPipelinesService.getPipelineStatus(body);
|
||||
const response = await this.pipelinesClientService.getPipelineStatus(body);
|
||||
|
||||
return response;
|
||||
}
|
||||
@@ -247,6 +246,31 @@ export class PipelinesController {
|
||||
return pipelineRes;
|
||||
}
|
||||
|
||||
@Get("/:id/data-assets")
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET
|
||||
)
|
||||
async findAllDataAssetByPipeline(
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
@Query('object') object: string
|
||||
) {
|
||||
const payload = {
|
||||
pipeline: id,
|
||||
object: object,
|
||||
};
|
||||
|
||||
this.logger.info(`GET pipelinesV2/:id/data-assets` + JSON.stringify(payload));
|
||||
|
||||
const result =
|
||||
await this.pipelinesClientService.findAllDataAssetByPipeline(payload, user);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@Patch('/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async update(
|
||||
@@ -342,6 +366,21 @@ export class PipelinesController {
|
||||
return response;
|
||||
}
|
||||
|
||||
@Patch('/:id/upgrade')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
async upgradeConnector(
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser
|
||||
) {
|
||||
this.logger.info('PipelinesController - upgrade connector');
|
||||
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
await this.pipelinesClientService.upgrade(id, metadata);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@ApiNoContentResponse()
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@@ -471,7 +510,7 @@ export class PipelinesController {
|
||||
},
|
||||
);
|
||||
|
||||
const response = await this.oldPipelinesService.runPipeline({ id, info });
|
||||
const response = await this.pipelinesClientService.runPipeline({ id, info });
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
@@ -7,25 +7,25 @@ import { PipelinesService } from './pipelines.service';
|
||||
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
|
||||
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||
import { ConnectorModule } from '../connector/connector.module';
|
||||
import { InputsModule } from '../inputs/inputs.module';
|
||||
import { TransformationsModule } from '../transformations/transformations.module';
|
||||
import { PlatformApiModule } from '../platform-api/platform-api.module';
|
||||
import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
import { CatalogModule } from '../catalog/catalog.module';
|
||||
|
||||
const client = new PipelinesClientConfiguration();
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ClientsModule.register([client.providerOptions]),
|
||||
OldPipelineModule,
|
||||
ConnectorModule,
|
||||
InputsModule,
|
||||
TransformationsModule,
|
||||
PlatformApiModule,
|
||||
NimbusServicesModule
|
||||
NimbusServicesModule,
|
||||
CatalogModule
|
||||
],
|
||||
controllers: [PipelinesController],
|
||||
providers: [PipelinesService, DadosferaLogger, NimbusService],
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/* eslint-disable no-async-promise-executor */
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Inject,
|
||||
@@ -17,7 +18,7 @@ import { lastValueFrom } from 'rxjs';
|
||||
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { ConnectorClientService } from '../connector/client.service';
|
||||
@@ -32,6 +33,10 @@ import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entitie
|
||||
import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { AxiosError } from 'axios';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { IDataAsset } from '../catalog/dtos';
|
||||
import { CatalogService } from '../catalog/catalog.service';
|
||||
|
||||
type RollbackPromise = () => Promise<any>;
|
||||
|
||||
@@ -48,7 +53,8 @@ export class PipelinesService implements OnModuleInit {
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly transformationsService: TransformationsService,
|
||||
private readonly platformAPI: PlatformApiService,
|
||||
private readonly nimbusService: NimbusService
|
||||
private readonly nimbusService: NimbusService,
|
||||
private readonly catalogService: CatalogService
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
@@ -171,6 +177,15 @@ export class PipelinesService implements OnModuleInit {
|
||||
return updatePipelineResponse;
|
||||
}
|
||||
|
||||
async upgrade(id: string, metadata: Metadata) {
|
||||
await lastValueFrom(
|
||||
this.pipelineWriteService.Upgrade(
|
||||
{ id },
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
|
||||
const { id, metadata, user } = data;
|
||||
const info = {
|
||||
@@ -577,4 +592,90 @@ export class PipelinesService implements OnModuleInit {
|
||||
this.logger.info('Platform api response: ' + JSON.stringify(response));
|
||||
}
|
||||
|
||||
async findAllDataAssetByPipeline(data: {
|
||||
pipeline: string,
|
||||
object?: string
|
||||
}, user: RequestUser) {
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
const isDataAdmin = user.permissions.includes(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
|
||||
);
|
||||
|
||||
let has_permission = false;
|
||||
|
||||
const {
|
||||
data_assets: resultString
|
||||
} = await lastValueFrom(
|
||||
this.pipelineReadService.FindAllDataAssetByPipeline(data, metadata)
|
||||
);
|
||||
|
||||
const result = JSON.parse(resultString) as any;
|
||||
const data_assets: IDataAsset[] = []
|
||||
result.forEach(data_asset => {
|
||||
if (data_asset?.owner === user.username) has_permission = true;
|
||||
|
||||
for (const role of user.roles) {
|
||||
if (data_asset.roles.includes(role)) has_permission = true;
|
||||
}
|
||||
|
||||
if (data_asset.users.includes(user.user_id)) has_permission = true;
|
||||
|
||||
if (isDataAdmin || has_permission) {
|
||||
delete data_asset.p_roles;
|
||||
delete data_asset.p_users;
|
||||
data_assets.push(data_asset as IDataAsset);
|
||||
}
|
||||
});
|
||||
|
||||
const assets = await this.catalogService.getAssetsUsersAndRoles(data_assets, user.customer_id);
|
||||
|
||||
return assets;
|
||||
}
|
||||
|
||||
async getPipelineStatus(data) {
|
||||
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
||||
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
|
||||
)
|
||||
.then((res) => {
|
||||
const statusArray =
|
||||
res.status?.sort((a, b) => {
|
||||
if (a.id < b.id) {
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}) || [];
|
||||
return { status: statusArray };
|
||||
})
|
||||
.catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
this.logger.info('Done');
|
||||
|
||||
return statusPipelineResponse;
|
||||
}
|
||||
|
||||
async runPipeline({ id, info }: IIdRequest) {
|
||||
this.logger.info('PipelinesClientService - RunPipeline');
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
|
||||
).catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
|
||||
if (statusPipelineResponse.status == false) {
|
||||
throw new ConflictException(
|
||||
'This pipeline is not ready yet to execute, Try again later!',
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.info('Done');
|
||||
return statusPipelineResponse;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
NotFoundException,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import {
|
||||
@@ -72,6 +72,10 @@ export class PlatformApiController {
|
||||
return id?.replace(/-/g, '_') || '';
|
||||
}
|
||||
|
||||
private decodePathParam(value: string): string {
|
||||
return value ? decodeURIComponent(value) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Denormalize ID back to UUID format (replace _ with -).
|
||||
* Used when we receive a normalized ID but need the original UUID.
|
||||
@@ -832,6 +836,40 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
|
||||
@ApiOperation({
|
||||
summary: 'Get pipeline run jobs',
|
||||
description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.',
|
||||
})
|
||||
@ApiOkResponse({
|
||||
description: 'DB-backed job runs for the selected pipeline run.',
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
jobs: {
|
||||
type: 'array',
|
||||
items: { type: 'object' },
|
||||
},
|
||||
},
|
||||
required: ['jobs'],
|
||||
},
|
||||
})
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineRunJobs(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('runId') runId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const decodedRunId = this.decodePathParam(runId);
|
||||
|
||||
return this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== JOBS - COLUMN EDITING ROUTES ====================
|
||||
|
||||
@Put('jobs/:jobId/input')
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
export type ReleaseNoteDTO = {
|
||||
id: string;
|
||||
date: string;
|
||||
tag: string;
|
||||
title: string;
|
||||
visible: boolean;
|
||||
expiryDate: string;
|
||||
content: string;
|
||||
showEmojis: boolean;
|
||||
image?: string;
|
||||
link?: string;
|
||||
linkText?: string;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { ReleaseNoteController } from './release_note.controller';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
|
||||
describe('ReleaseNoteController', () => {
|
||||
let controller: ReleaseNoteController;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
controllers: [ReleaseNoteController],
|
||||
providers: [ReleaseNoteService],
|
||||
}).compile();
|
||||
|
||||
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(controller).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
import { Controller, Get, Inject } from '@nestjs/common';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Controller('release_note')
|
||||
@Authenticated()
|
||||
export class ReleaseNoteController {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private readonly releaseNoteService: ReleaseNoteService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Get()
|
||||
async getLatestReleaseNote(@Language() language: LanguageEnum) {
|
||||
this.logger.info(`Fetching latest release note for language: ${language}`);
|
||||
return await this.releaseNoteService.getLatestReleaseNote(language);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
import { ReleaseNoteController } from './release_note.controller';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Module({
|
||||
controllers: [ReleaseNoteController],
|
||||
providers: [ReleaseNoteService, DadosferaLogger]
|
||||
})
|
||||
export class ReleaseNoteModule {}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
|
||||
describe('ReleaseNoteService', () => {
|
||||
let service: ReleaseNoteService;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [ReleaseNoteService],
|
||||
}).compile();
|
||||
|
||||
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import axios, { AxiosInstance } from 'axios';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { ReleaseNoteDTO } from './dto/release_note.dto';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Injectable()
|
||||
export class ReleaseNoteService {
|
||||
client: AxiosInstance;
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
this.client = axios.create({
|
||||
baseURL: process.env.FIREBASE_BASE_URL,
|
||||
});
|
||||
}
|
||||
|
||||
async getLatestReleaseNote(lang: LanguageEnum) {
|
||||
try {
|
||||
const lng = lang.split('-');
|
||||
const language = lng[0] + '-' + lng[1].toUpperCase();
|
||||
|
||||
const endpoint = `/release_note/${language}.json`;
|
||||
const {
|
||||
data,
|
||||
status,
|
||||
config
|
||||
} = await this.client.get<ReleaseNoteDTO>(endpoint)
|
||||
this.logger.info(`Fetched release note for language: ${lang} with status: ${status}`);
|
||||
this.logger.info(`Request URL: ${config.baseURL}/${config.url}`);
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
this.logger.error(`Error fetching release note: ${error.message}`);
|
||||
|
||||
if (axios.isAxiosError(error)) {
|
||||
this.logger.error(`Axios error details: ${error.toJSON()}`);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+5
-2
@@ -1,5 +1,8 @@
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export interface Info {
|
||||
user_id: string;
|
||||
customer_id: string;
|
||||
customer: string;
|
||||
}
|
||||
export interface ICreateTransformationsRequest {
|
||||
transformations: Transformation[];
|
||||
info: Info;
|
||||
|
||||
Reference in New Issue
Block a user