mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-01 12:18:15 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
095306d2c7 |
@@ -71,11 +71,6 @@ jobs:
|
||||
sudo mv helmfile /usr/local/bin/
|
||||
helmfile --version
|
||||
|
||||
- name: Install Helm Diff plugin
|
||||
run: |
|
||||
helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
|
||||
helm diff version
|
||||
|
||||
- name: Debug Helm env
|
||||
run: |
|
||||
helm env
|
||||
@@ -107,5 +102,4 @@ jobs:
|
||||
- name: Run Helmfile Diff
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
HELM_PLUGINS: /home/runner/.local/share/helm/plugins
|
||||
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
FROM node:20-alpine AS base_image
|
||||
RUN npm install -g npm@10.8.2
|
||||
RUN npm install -g npm@latest
|
||||
|
||||
FROM base_image AS build_base
|
||||
WORKDIR /app
|
||||
@@ -22,7 +22,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
# run aws cli without mounting secret, because CI already has AWS credentials
|
||||
FROM build_base AS ci_image
|
||||
RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||
RUN npm ci --ignore-scripts
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ FROM build_base AS dev
|
||||
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
||||
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||
# flag --build-from-source is required to force-build sqlite3
|
||||
RUN npm ci --ignore-scripts
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
ENTRYPOINT npm run start:dev
|
||||
|
||||
|
||||
+1
-1
@@ -22,7 +22,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
FROM build_base AS build
|
||||
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
||||
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||
RUN npm ci --ignore-scripts
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
# Maestro
|
||||
|
||||
|
||||
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
|
||||
|
||||
```mermaid
|
||||
|
||||
@@ -111,12 +111,14 @@ spec:
|
||||
value: "{{ .Values.maestro.redis_tls }}"
|
||||
- name: PLATFORM_API_URL
|
||||
value: {{ .Values.maestro.platform_api_url }}
|
||||
- name: CONNECTIONS_API_URL
|
||||
value: {{ .Values.maestro.connections_api_url | default "" | quote }}
|
||||
- name: STORAGE_EXPLORER_API_URL
|
||||
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
|
||||
- name: FIREBASE_BASE_URL
|
||||
value: {{ .Values.maestro.firebase_base_url }}
|
||||
- name: AUTODRIVE_EXTRACTOR_API_URL
|
||||
value: {{ .Values.maestro.autodrive_extractor_api_url | quote }}
|
||||
- name: AUTODRIVE_CORE_API_URL
|
||||
value: {{ .Values.maestro.autodrive_core_api_url | quote }}
|
||||
- name: AUTODRIVE_ASSISTANT_API_URL
|
||||
value: {{ .Values.maestro.autodrive_assistant_api_url | quote }}
|
||||
- name: JWT_PRIVATE_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -9,9 +9,7 @@ maestro:
|
||||
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
|
||||
redis_database: "1"
|
||||
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
|
||||
connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com
|
||||
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
|
||||
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
|
||||
|
||||
hostname: maestro.stg.dadosfera.ai
|
||||
|
||||
|
||||
@@ -48,6 +48,9 @@ maestro:
|
||||
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
||||
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
|
||||
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
|
||||
autodrive_extractor_api_url: "https://autodrive-extractor-api-{customer}.dadosfera.ai"
|
||||
autodrive_core_api_url: "https://autodrive-api-{customer}.dadosfera.ai"
|
||||
autodrive_assistant_api_url: "https://autodrive-assistant-api-{customer}.dadosfera.ai"
|
||||
dedicated_proxy: ""
|
||||
restricted_ip: ""
|
||||
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
|
||||
@@ -55,7 +58,6 @@ maestro:
|
||||
redis_database: "0"
|
||||
redis_tls: "true"
|
||||
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
|
||||
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
|
||||
+296
-516
File diff suppressed because it is too large
Load Diff
Generated
+21
-83
@@ -16,7 +16,8 @@
|
||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "3.38.0-beta.28",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -30,7 +31,7 @@
|
||||
"@nestjs/schematics": "^9.2.0",
|
||||
"@nestjs/swagger": "^6.3.0",
|
||||
"@nestjs/testing": "^9.4.3",
|
||||
"axios": "0.30.3",
|
||||
"axios": "^0.30.2",
|
||||
"cache-manager": "^5.1.4",
|
||||
"cache-manager-ioredis-yet": "^1.1.0",
|
||||
"class-transformer": "^0.5.1",
|
||||
@@ -46,7 +47,6 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
@@ -1734,10 +1734,20 @@
|
||||
"winston-log2gelf": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack": {
|
||||
"version": "2.5.3",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
|
||||
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"rxjs": "^7.5.5"
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.40.0-beta.14",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
|
||||
"integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
|
||||
"version": "3.38.0-beta.28",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
|
||||
"integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"rxjs": "^7.5.5"
|
||||
@@ -2918,20 +2928,6 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
|
||||
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
|
||||
"engines": [
|
||||
"node >= 0.8"
|
||||
],
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^2.2.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
|
||||
"version": "0.5.4",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
|
||||
@@ -3101,24 +3097,6 @@
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/multer": {
|
||||
"version": "1.4.4-lts.1",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
|
||||
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
|
||||
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.0.0",
|
||||
"concat-stream": "^1.5.2",
|
||||
"mkdirp": "^0.5.4",
|
||||
"object-assign": "^4.1.1",
|
||||
"type-is": "^1.6.4",
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
|
||||
@@ -3143,25 +3121,6 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
|
||||
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
|
||||
"dependencies": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
@@ -3236,19 +3195,6 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"dependencies": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/tslib": {
|
||||
"version": "2.5.3",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
|
||||
@@ -5588,9 +5534,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "0.30.3",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.3.tgz",
|
||||
"integrity": "sha512-5/tmEb6TmE/ax3mdXBc/Mi6YdPGxQsv+0p5YlciXWt3PHIn0VamqCXhRMtScnwY3lbgSXLneOuXAKUhgmSRpwg==",
|
||||
"version": "0.30.2",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.2.tgz",
|
||||
"integrity": "sha512-0pE4RQ4UQi1jKY6p7u6i1Tkzqmu+d+/tHS7Q7rKunWLB9WyilBTpHHpXzPNMDj5hTbK0B0PTLSz07yqMBiF6xg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.4",
|
||||
@@ -6563,6 +6509,7 @@
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
@@ -9181,11 +9128,6 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
|
||||
},
|
||||
"node_modules/isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
@@ -10783,6 +10725,7 @@
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
|
||||
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.6.0",
|
||||
@@ -11785,11 +11728,6 @@
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"node_modules/process-warning": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
|
||||
|
||||
+5
-8
@@ -10,7 +10,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
|
||||
"prebuild": "rimraf dist",
|
||||
"build": "nest build",
|
||||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
||||
@@ -34,7 +34,8 @@
|
||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "3.38.0-beta.28",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -48,7 +49,7 @@
|
||||
"@nestjs/schematics": "^9.2.0",
|
||||
"@nestjs/swagger": "^6.3.0",
|
||||
"@nestjs/testing": "^9.4.3",
|
||||
"axios": "0.30.3",
|
||||
"axios": "^0.30.2",
|
||||
"cache-manager": "^5.1.4",
|
||||
"cache-manager-ioredis-yet": "^1.1.0",
|
||||
"class-transformer": "^0.5.1",
|
||||
@@ -64,7 +65,6 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
@@ -80,7 +80,7 @@
|
||||
"swagger-ui-express": "^4.6.3"
|
||||
},
|
||||
"overrides": {
|
||||
"axios": "0.30.3",
|
||||
"multer": "2.0.2",
|
||||
"form-data": "^4.0.4",
|
||||
"body-parser": "^1.20.3",
|
||||
"cross-spawn": "^7.0.5",
|
||||
@@ -117,8 +117,5 @@
|
||||
"ts-node": "^10.9.1",
|
||||
"tsconfig-paths": "^3.14.2",
|
||||
"typescript": "^4.9.5"
|
||||
},
|
||||
"resolutions": {
|
||||
"axios": "0.30.3"
|
||||
}
|
||||
}
|
||||
|
||||
+8
-3
@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
|
||||
import { NetworkConfigModule } from './modules/network-config/network-config.module';
|
||||
import { InputsModule } from './modules/inputs/inputs.module';
|
||||
import { OauthModule } from './modules/oauth/oauth.module';
|
||||
import { PipelinesModule } from './modules/pipelines/pipelines.module';
|
||||
import { TransformationsModule } from './modules/transformations/transformations.module';
|
||||
import { HealthModule } from './modules/health/health.module';
|
||||
import { CatalogModule } from './modules/catalog/catalog.module';
|
||||
@@ -34,8 +35,9 @@ import { ShareMetadataModule } from './modules/share-metadata/share-metadata.mod
|
||||
import { ApiKeyModule } from './modules/api-key/api-key.module';
|
||||
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
|
||||
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
|
||||
import { ReleaseNoteModule } from './modules/release_note/release_note.module';
|
||||
|
||||
import { AutodriveExtractorModule } from './modules/autodrive-extractor/autodrive-extractor.module';
|
||||
import { AutodriveCoreModule } from './modules/autodrive-core/autodrive-core.module';
|
||||
import { AutodriveAssistantModule } from './modules/autodrive-assistant/autodrive-assistant.module';
|
||||
|
||||
@Module({
|
||||
providers: [
|
||||
@@ -59,6 +61,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
|
||||
PermissionsModule,
|
||||
TermsOfUseModule,
|
||||
ConnectionTestModule,
|
||||
PipelinesModule,
|
||||
TransformationsModule,
|
||||
UsersModule,
|
||||
RolesModule,
|
||||
@@ -77,9 +80,11 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
|
||||
NetworkPolicyModule,
|
||||
PlatformApiModule,
|
||||
StorageExplorerModule,
|
||||
AutodriveExtractorModule,
|
||||
AutodriveCoreModule,
|
||||
AutodriveAssistantModule,
|
||||
//Always leave HealthModule last, so it is on the bottom of swagger
|
||||
HealthModule,
|
||||
ReleaseNoteModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -153,7 +153,6 @@ export class AuthenticationGuard
|
||||
user_id: accessTokenPayload.user_id,
|
||||
username: accessTokenPayload.username,
|
||||
permissions: accessTokenPayload.permissions,
|
||||
roles: accessTokenPayload.roles,
|
||||
customer_id: accessTokenPayload.customer_id,
|
||||
customer_name: accessTokenPayload.customer_name,
|
||||
customer_tier: accessTokenPayload.customer_tier,
|
||||
|
||||
@@ -11,7 +11,6 @@ export function extractUserFrom(aRawJwt: string) {
|
||||
user_id: payload.user_id,
|
||||
username: payload.username,
|
||||
permissions: payload.permissions,
|
||||
roles: payload.roles,
|
||||
customer_id: payload.customer_id,
|
||||
customer_name: payload.customer_name,
|
||||
customer_tier: payload.customer_tier,
|
||||
|
||||
@@ -357,16 +357,6 @@ export const PERMISSIONS_GROUPS = {
|
||||
'es-es': 'Crear y editar atributos en el catálogo',
|
||||
},
|
||||
},
|
||||
CERTIFY: {
|
||||
seqid: 53,
|
||||
claim: 'catalog:certify',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Alterar o status de certificação dos Ativos',
|
||||
'en-us': "Change Assets' certification status",
|
||||
'es-es': 'Cambiar el estado de certificación de los Activos',
|
||||
},
|
||||
},
|
||||
DELETE: {
|
||||
seqid: 1,
|
||||
claim: 'catalog:delete',
|
||||
@@ -707,6 +697,93 @@ export const PERMISSIONS_GROUPS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_EXTRACTOR: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Extractor',
|
||||
'en-us': 'Autodrive Extractor',
|
||||
'es-es': 'Autodrive Extractor',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 53,
|
||||
claim: 'autodrive-extractor:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Extractor',
|
||||
'en-us': 'Read Autodrive Extractor data',
|
||||
'es-es': 'Leer datos del Autodrive Extractor',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 54,
|
||||
claim: 'autodrive-extractor:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Extractor',
|
||||
'en-us': 'Write Autodrive Extractor data',
|
||||
'es-es': 'Escribir datos en Autodrive Extractor',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_CORE: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Core',
|
||||
'en-us': 'Autodrive Core',
|
||||
'es-es': 'Autodrive Core',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 55,
|
||||
claim: 'autodrive-core:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Core',
|
||||
'en-us': 'Read Autodrive Core data',
|
||||
'es-es': 'Leer datos del Autodrive Core',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 56,
|
||||
claim: 'autodrive-core:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Core',
|
||||
'en-us': 'Write Autodrive Core data',
|
||||
'es-es': 'Escribir datos en Autodrive Core',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_ASSISTANT: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Assistant',
|
||||
'en-us': 'Autodrive Assistant',
|
||||
'es-es': 'Autodrive Assistant',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 57,
|
||||
claim: 'autodrive-assistant:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Assistant',
|
||||
'en-us': 'Read Autodrive Assistant data',
|
||||
'es-es': 'Leer datos del Autodrive Assistant',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 58,
|
||||
claim: 'autodrive-assistant:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Assistant',
|
||||
'en-us': 'Write Autodrive Assistant data',
|
||||
'es-es': 'Escribir datos en Autodrive Assistant',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
export interface DadosferaModule {
|
||||
name: string;
|
||||
|
||||
@@ -12,7 +12,6 @@ export interface RequestUser {
|
||||
customer_tier: string;
|
||||
access_token: string;
|
||||
customer_modules: string[];
|
||||
roles: string[];
|
||||
}
|
||||
|
||||
export const User: (options?: { required?: boolean }) => ParameterDecorator =
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
Inject,
|
||||
Injectable,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { map, Observable } from 'rxjs';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import {
|
||||
ReadService,
|
||||
ProtoServices,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
|
||||
import { PipelinesClientConfiguration } from 'src/modules/pipelinesV2/pipelines-client';
|
||||
import { PlatformApiService } from 'src/modules/platform-api/platform-api.service';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Injectable()
|
||||
export class PipelineExecutionGuard implements CanActivate {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private readonly platformApiService: PlatformApiService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
try {
|
||||
this.logger.info(
|
||||
'PipelineExecutionGuard: Checking if pipeline can be executed...',
|
||||
);
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const pipelineId = request.params.pipelineId;
|
||||
const user = request.user;
|
||||
const idRegex = /[^0-9a-zA-Z_$]+/g;
|
||||
const convertedId = pipelineId.replace(idRegex, '_');
|
||||
|
||||
const status = await this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/pipeline/${convertedId}/pipeline_run`,
|
||||
user,
|
||||
);
|
||||
|
||||
const currentStatus = status[status.length - 1]
|
||||
|
||||
this.logger.info('Pipeline current status response:' + JSON.stringify(currentStatus));
|
||||
|
||||
if (currentStatus.last_status.toLowerCase() === 'running') {
|
||||
this.logger.error('Pipeline is running, cannot update input now');
|
||||
throw new BadRequestException('Pipeline is running, cannot update input now');
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error('Error in PipelineExecutionGuard: ' + error.message);
|
||||
throw new BadRequestException('Error checking pipeline status: ' + error.message);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -111,4 +111,3 @@ function configureSwagger(app: INestApplication) {
|
||||
);
|
||||
}
|
||||
bootstrap();
|
||||
|
||||
|
||||
@@ -478,7 +478,6 @@ export class AuthController {
|
||||
@Get('me')
|
||||
async getMe(@Req() req: Request, @Res() res: Response) {
|
||||
this.logger.info('GET /auth/me ')
|
||||
this.logger.info(JSON.stringify(req.headers));
|
||||
|
||||
// Check for API key header first
|
||||
const apiKey = req.get('X-Api-key');
|
||||
@@ -489,7 +488,6 @@ export class AuthController {
|
||||
const userDto = {
|
||||
id: api_key.user_id,
|
||||
name: api_key.username,
|
||||
email: api_key.username,
|
||||
customer: {
|
||||
id: api_key.customer_id,
|
||||
name: api_key.customer_name,
|
||||
@@ -504,7 +502,7 @@ export class AuthController {
|
||||
const accessToken = req.cookies['ddf-auth'];
|
||||
const refreshToken = req.cookies['ddf-refresh-auth'];
|
||||
const userId = req.cookies['ddf-user-id'];
|
||||
const resourceHost = req.headers["x-original-url"] as string || "" ;
|
||||
const resourceHost = req.headers["host"]
|
||||
|
||||
const hasUserSession = Boolean(accessToken) && Boolean(userId);
|
||||
this.logger.info('Has User Session: ' + hasUserSession);
|
||||
|
||||
@@ -437,8 +437,7 @@ export class AuthClientService implements OnModuleInit {
|
||||
|
||||
const userDto: UserDTO = {
|
||||
id: user.id,
|
||||
name: user.name,
|
||||
email: user.email,
|
||||
name: user.username,
|
||||
jobTitle: user?.jobTitle || null,
|
||||
department: user?.department || null,
|
||||
hierarchy: user?.hierarchy || null,
|
||||
|
||||
@@ -144,7 +144,6 @@ export interface BulkEditResponse {
|
||||
export type UserDTO = {
|
||||
id: string,
|
||||
name: string,
|
||||
email: string,
|
||||
jobTitle?: string,
|
||||
department?: string,
|
||||
hierarchy?: string,
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_ASSISTANT_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_ASSISTANT_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_ASSISTANT_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_ASSISTANT_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,201 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveAssistantService } from './autodrive-assistant.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Assistant')
|
||||
@Controller('autodrive-assistant')
|
||||
@Authenticated()
|
||||
export class AutodriveAssistantController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveAssistantService: AutodriveAssistantService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// ASSISTANTS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List assistants' })
|
||||
@Get('assistants')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async listAssistants(
|
||||
@Query('limit') limit: number,
|
||||
@Query('offset') offset: number,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
'/assistants',
|
||||
user,
|
||||
undefined,
|
||||
{ limit, offset },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create assistant' })
|
||||
@Post('assistants')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async createAssistant(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'POST',
|
||||
'/assistants',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get assistant by ID' })
|
||||
@Get('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async getAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update assistant' })
|
||||
@Put('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async updateAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'PUT',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete assistant' })
|
||||
@Delete('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async deleteAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'DELETE',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// KNOWLEDGE BASE
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List knowledge bases linked to assistant' })
|
||||
@Get('assistants/:assistantId/knowledge-bases')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async listKnowledgeBases(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/assistants/${assistantId}/knowledge-bases`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update knowledge base associations for assistant' })
|
||||
@Put('assistants/:assistantId/knowledge-bases')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async updateKnowledgeBases(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'PUT',
|
||||
`/assistants/${assistantId}/knowledge-bases`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// QUESTIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Ask question via assistant' })
|
||||
@Post('dataset/:datasetId/ai_question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async aiQuestion(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/ai_question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get AI question answer' })
|
||||
@Get('dataset/:datasetId/ai_question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async getAiQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/ai_question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UTILITY
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveAssistantService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveAssistantService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveAssistantController } from './autodrive-assistant.controller';
|
||||
import { AutodriveAssistantService } from './autodrive-assistant.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveAssistantController],
|
||||
providers: [AutodriveAssistantService, DadosferaLogger],
|
||||
exports: [AutodriveAssistantService],
|
||||
})
|
||||
export class AutodriveAssistantModule {}
|
||||
+32
-37
@@ -1,26 +1,18 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import { SignatureV4 } from '@aws-sdk/signature-v4';
|
||||
import { Sha256 } from '@aws-crypto/sha256-js';
|
||||
import { defaultProvider } from '@aws-sdk/credential-provider-node';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { CONNECTIONS_API_CONFIG } from './connections-api.config';
|
||||
import { AUTODRIVE_ASSISTANT_CONFIG } from './autodrive-assistant.config';
|
||||
|
||||
@Injectable()
|
||||
export class ConnectionsApiService {
|
||||
private signer: SignatureV4;
|
||||
export class AutodriveAssistantService {
|
||||
private logger: any;
|
||||
|
||||
constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) {
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
this.signer = new SignatureV4({
|
||||
service: 'execute-api',
|
||||
region: CONNECTIONS_API_CONFIG.region,
|
||||
credentials: defaultProvider(),
|
||||
sha256: Sha256,
|
||||
});
|
||||
}
|
||||
|
||||
async proxy(
|
||||
@@ -28,9 +20,13 @@ export class ConnectionsApiService {
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, string>,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
const baseUrl = CONNECTIONS_API_CONFIG.getUrl();
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive assistant operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_ASSISTANT_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
@@ -40,59 +36,58 @@ export class ConnectionsApiService {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
host: url.hostname,
|
||||
'content-type': 'application/json',
|
||||
customer_name: user.customer_name || '',
|
||||
customer_id: user.customer_id || '',
|
||||
'x-user-id': user.user_id || '',
|
||||
'x-username': user.username || '',
|
||||
'x-customer-tier': user.customer_tier || '',
|
||||
'x-customer-id': user.customer_id || '',
|
||||
};
|
||||
const requestToSign = {
|
||||
method: method.toUpperCase(),
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: url.port ? parseInt(url.port, 10) : undefined,
|
||||
path: url.pathname + url.search,
|
||||
headers,
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-assistant', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const signedRequest = await this.signer.sign(requestToSign);
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers: signedRequest.headers as Record<string, string>,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: CONNECTIONS_API_CONFIG.timeout,
|
||||
timeout: AUTODRIVE_ASSISTANT_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Connections API proxy error', {
|
||||
this.logger.error('Autodrive Assistant API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Connections API service unavailable', 503);
|
||||
throw new HttpException('Autodrive Assistant API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Connections API request timeout', 504);
|
||||
throw new HttpException('Autodrive Assistant API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_CORE_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_CORE_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_CORE_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_CORE_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,286 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
UseInterceptors,
|
||||
UploadedFiles,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
|
||||
import { FilesInterceptor } from '@nestjs/platform-express';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import FormData from 'form-data';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveCoreService } from './autodrive-core.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Core')
|
||||
@Controller('autodrive-core')
|
||||
@Authenticated()
|
||||
export class AutodriveCoreController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveCoreService: AutodriveCoreService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DATASET MANAGEMENT
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List datasets' })
|
||||
@Get('datasets')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async listDatasets(
|
||||
@Query('name') name: string,
|
||||
@Query('limit') limit: number,
|
||||
@Query('offset') offset: number,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
'/datasets',
|
||||
user,
|
||||
undefined,
|
||||
{ name, limit, offset },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get dataset by ID' })
|
||||
@Get('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Add documents to existing dataset' })
|
||||
@Put('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async updateDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'PUT',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete dataset' })
|
||||
@Delete('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async deleteDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'DELETE',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Append documents from URLs to dataset' })
|
||||
@Post('dataset/:datasetId/append-from-url')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async appendFromUrl(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/append-from-url`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get dataset creation/update logs' })
|
||||
@Get('dataset/:datasetId/logs')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getDatasetLogs(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/logs`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// FILE OPERATIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Create vector dataset from uploaded files' })
|
||||
@Post('upload')
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@UseInterceptors(FilesInterceptor('files'))
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async upload(
|
||||
@UploadedFiles() files: Array<Express.Multer.File>,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const formData = new FormData();
|
||||
if (files && files.length > 0) {
|
||||
files.forEach((file) => {
|
||||
formData.append('files', file.buffer, {
|
||||
filename: file.originalname,
|
||||
contentType: file.mimetype,
|
||||
});
|
||||
});
|
||||
}
|
||||
// Forward additional body fields
|
||||
if (body) {
|
||||
Object.entries(body).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null && key !== 'files') {
|
||||
formData.append(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
return this.autodriveCoreService.proxyFormData(
|
||||
'POST',
|
||||
'/upload',
|
||||
user,
|
||||
formData,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create vector dataset from URLs' })
|
||||
@Post('upload-from-url')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async uploadFromUrl(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
'/upload-from-url',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// QUERY OPERATIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Search dataset using semantic search' })
|
||||
@Post('dataset/:datasetId/question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async question(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get semantic search results' })
|
||||
@Get('dataset/:datasetId/question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Ask AI question on dataset' })
|
||||
@Post('dataset/:datasetId/ai_question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async aiQuestion(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/ai_question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get AI question answer' })
|
||||
@Get('dataset/:datasetId/ai_question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getAiQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/ai_question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// SYSTEM
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get frontend configuration' })
|
||||
@Get('config')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getConfig(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/config', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get usage metrics' })
|
||||
@Get('usage')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getUsage(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/usage', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveCoreController } from './autodrive-core.controller';
|
||||
import { AutodriveCoreService } from './autodrive-core.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveCoreController],
|
||||
providers: [AutodriveCoreService, DadosferaLogger],
|
||||
exports: [AutodriveCoreService],
|
||||
})
|
||||
export class AutodriveCoreModule {}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_CORE_CONFIG } from './autodrive-core.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveCoreService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive core operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_CORE_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-core', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_CORE_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Core API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Core API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Core API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
|
||||
async proxyFormData(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
formData: any,
|
||||
query?: Record<string, any>,
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive core operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_CORE_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
...formData.getHeaders?.(),
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying form data request to autodrive-core', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: formData,
|
||||
timeout: AUTODRIVE_CORE_CONFIG.timeout,
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Core API form data proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_EXTRACTOR_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_EXTRACTOR_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_EXTRACTOR_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_EXTRACTOR_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,491 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
UseInterceptors,
|
||||
UploadedFiles,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
|
||||
import { FilesInterceptor } from '@nestjs/platform-express';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import FormData from 'form-data';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveExtractorService } from './autodrive-extractor.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Extractor')
|
||||
@Controller('autodrive-extractor')
|
||||
@Authenticated()
|
||||
export class AutodriveExtractorController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveExtractorService: AutodriveExtractorService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LEGACY - NPL Extraction routes (to be removed)
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Extract features from dataset using NPL' })
|
||||
@Post('npl/datasets/:datasetId/extract')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplExtract(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
`/npl/datasets/${datasetId}/extract`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] List NPL extractions for a dataset' })
|
||||
@Get('npl/datasets/:datasetId/extractions')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplListExtractions(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Query('status') status: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions`,
|
||||
user,
|
||||
undefined,
|
||||
{ status },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL extraction details' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetExtraction(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL extraction to CSV' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportExtractionCsv(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL extraction to XLSX' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/xlsx')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportExtractionXlsx(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/xlsx`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Batch export all NPL extractions to CSV' })
|
||||
@Get('npl/datasets/:datasetId/extractions/export/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplBatchExportCsv(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/export/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Batch export all NPL extractions to XLSX' })
|
||||
@Get('npl/datasets/:datasetId/extractions/export/xlsx')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplBatchExportXlsx(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/export/xlsx`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL extraction report' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/report')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetExtractionReport(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/report`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LEGACY - NPL Configuration routes (to be removed)
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration' })
|
||||
@Get('npl/config')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration summary' })
|
||||
@Get('npl/config/summary')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigSummary(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/summary', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] List NPL configuration fields' })
|
||||
@Get('npl/config/fields')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigFields(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/fields', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration field by ID' })
|
||||
@Get('npl/config/fields/:fieldId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigField(
|
||||
@Param('fieldId') fieldId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/config/fields/${fieldId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Update NPL configuration field' })
|
||||
@Put('npl/config/fields/:fieldId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplUpdateConfigField(
|
||||
@Param('fieldId') fieldId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
`/npl/config/fields/${fieldId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL RAG queries' })
|
||||
@Get('npl/config/rag-queries')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetRagQueries(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/rag-queries', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL system prompt' })
|
||||
@Get('npl/config/system-prompt')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetSystemPrompt(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/system-prompt', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Update NPL system prompt' })
|
||||
@Put('npl/config/system-prompt')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplUpdateSystemPrompt(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
'/npl/config/system-prompt',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL source priorities' })
|
||||
@Get('npl/config/source-priorities')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetSourcePriorities(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/source-priorities', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Reset NPL configuration to defaults' })
|
||||
@Post('npl/config/reset')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplResetConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('POST', '/npl/config/reset', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL configuration' })
|
||||
@Get('npl/config/export')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/export', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Import NPL configuration' })
|
||||
@Post('npl/config/import')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplImportConfig(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/npl/config/import',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// EXTRACTION TEMPLATES
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Seed NPL Brasil extraction template' })
|
||||
@Post('templates/seed-npl')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async seedNplTemplate(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/templates/seed-npl',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'List extraction templates' })
|
||||
@Get('templates')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async listTemplates(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/templates', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create extraction template' })
|
||||
@Post('templates')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async createTemplate(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/templates',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction template by ID' })
|
||||
@Get('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update extraction template' })
|
||||
@Put('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async updateTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete extraction template' })
|
||||
@Delete('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async deleteTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'DELETE',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Import extraction templates from file' })
|
||||
@Post('templates/import')
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@UseInterceptors(FilesInterceptor('file'))
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async importTemplates(
|
||||
@UploadedFiles() files: Array<Express.Multer.File>,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const formData = new FormData();
|
||||
if (files && files.length > 0) {
|
||||
files.forEach((file) => {
|
||||
formData.append('file', file.buffer, {
|
||||
filename: file.originalname,
|
||||
contentType: file.mimetype,
|
||||
});
|
||||
});
|
||||
}
|
||||
return this.autodriveExtractorService.proxyFormData(
|
||||
'POST',
|
||||
'/templates/import',
|
||||
user,
|
||||
formData,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// GENERIC EXTRACTION
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Start template-based extraction on dataset' })
|
||||
@Post('extraction/datasets/:datasetId/extract')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async startExtraction(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
`/extraction/datasets/${datasetId}/extract`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'List extraction jobs for a dataset' })
|
||||
@Get('extraction/datasets/:datasetId/jobs')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async listExtractionJobs(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Query('status') status: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/datasets/${datasetId}/jobs`,
|
||||
user,
|
||||
undefined,
|
||||
{ status },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction job details' })
|
||||
@Get('extraction/jobs/:jobId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getExtractionJob(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction job report' })
|
||||
@Get('extraction/jobs/:jobId/report')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getExtractionJobReport(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}/report`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Export extraction job to CSV' })
|
||||
@Get('extraction/jobs/:jobId/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async exportExtractionJobCsv(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UTILITY
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveExtractorController } from './autodrive-extractor.controller';
|
||||
import { AutodriveExtractorService } from './autodrive-extractor.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveExtractorController],
|
||||
providers: [AutodriveExtractorService, DadosferaLogger],
|
||||
exports: [AutodriveExtractorService],
|
||||
})
|
||||
export class AutodriveExtractorModule {}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_EXTRACTOR_CONFIG } from './autodrive-extractor.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveExtractorService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive extractor operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_EXTRACTOR_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-extractor', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_EXTRACTOR_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Extractor API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Extractor API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Extractor API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
|
||||
async proxyFormData(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
formData: any,
|
||||
query?: Record<string, any>,
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive extractor operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_EXTRACTOR_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
...formData.getHeaders?.(),
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying form data request to autodrive-extractor', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: formData,
|
||||
timeout: AUTODRIVE_EXTRACTOR_CONFIG.timeout,
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Extractor API form data proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ import {
|
||||
HttpStatus,
|
||||
Res,
|
||||
} from '@nestjs/common';
|
||||
import { ValidationPipe } from '../../pipes/object-validation.pipe';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
ApiHeaders,
|
||||
@@ -47,7 +46,6 @@ import {
|
||||
IMakeAComment,
|
||||
IOneDataAsset,
|
||||
IPreviewResponse,
|
||||
IUpdateCertificationStatusRequest,
|
||||
IUpdateDataRequest,
|
||||
TriggerCatalogReq,
|
||||
TriggerCatalogRes,
|
||||
@@ -243,51 +241,6 @@ export class CatalogController {
|
||||
return res;
|
||||
}
|
||||
|
||||
@Get('schemas')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async findSchemas(@User() user: RequestUser) {
|
||||
const { username, user_id, customer_id, customer_name } = user;
|
||||
this.logger.info(`/catalog - ON FIND SCHEMAS ROUTE`, {
|
||||
username,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
username,
|
||||
user_id,
|
||||
customer_id,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
try {
|
||||
const res = await this.catalogService.findSchemas(metadata);
|
||||
return res;
|
||||
} catch (error) {
|
||||
throw new HttpException(error.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Get('custom-properties')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async getCustomPropertyDefinitions(@User() user: RequestUser) {
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
});
|
||||
|
||||
return this.catalogService.getCustomPropertyDefinitions(metadata);
|
||||
}
|
||||
|
||||
@Get('data-asset/:id')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
@@ -512,8 +465,6 @@ export class CatalogController {
|
||||
language,
|
||||
});
|
||||
|
||||
delete (body as any).certification_status;
|
||||
|
||||
const result = await this.catalogService.updateOneDataAsset({
|
||||
body,
|
||||
data_asset_id,
|
||||
@@ -527,33 +478,6 @@ export class CatalogController {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Put('data-asset/:id/certification-status')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async updateDataAssetCertificationStatus(
|
||||
@User() user: RequestUser,
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') data_asset_id: string,
|
||||
@Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
|
||||
): Promise<IUpdateCertificationStatusRequest> {
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
language,
|
||||
});
|
||||
|
||||
return this.catalogService.updateCertificationStatus({
|
||||
body,
|
||||
data_asset_id,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
@Post('data-asset/:id/docs')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||
@@ -1019,4 +943,4 @@ export class CatalogController {
|
||||
this.logger.error(error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { CatalogController } from './catalog.controller';
|
||||
import { CatalogClientConfiguration } from './catalog-client';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||
import { UsersModule } from '../users/users.module';
|
||||
import { RolesModule } from '../roles/roles.module';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
import { ShareModule } from './share/share.module';
|
||||
import { CatalogService } from './catalog.service';
|
||||
import { MixpanelModule } from '../mixpanel/mixpanel.module';
|
||||
|
||||
const client = new CatalogClientConfiguration();
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ClientsModule.register([client.providerOptions]),
|
||||
OldPipelineModule,
|
||||
UsersModule,
|
||||
RolesModule,
|
||||
CustomersModule,
|
||||
|
||||
@@ -29,7 +29,6 @@ import {
|
||||
AssetReporter,
|
||||
BatchRemoveRlsRulesRequest,
|
||||
CreateDataDocsDTO,
|
||||
IUpdateCertificationStatusRequest,
|
||||
IUpdateDataRequest,
|
||||
TriggerCatalogReq,
|
||||
} from './dtos';
|
||||
@@ -120,10 +119,6 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async getCustomPropertyDefinitions(metadata: Metadata) {
|
||||
return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata));
|
||||
}
|
||||
|
||||
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
if (!data.embed) data.embed = undefined;
|
||||
@@ -219,6 +214,15 @@ class CatalogService implements OnModuleInit {
|
||||
|
||||
this.logger.debug('Extracted filters:', { filters });
|
||||
|
||||
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
|
||||
search,
|
||||
page,
|
||||
size,
|
||||
sort_by,
|
||||
order,
|
||||
filters,
|
||||
});
|
||||
|
||||
if (
|
||||
filters.manually !== undefined &&
|
||||
filters.manually !== null &&
|
||||
@@ -229,6 +233,15 @@ class CatalogService implements OnModuleInit {
|
||||
delete filters.manually;
|
||||
}
|
||||
|
||||
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
|
||||
search,
|
||||
page,
|
||||
size,
|
||||
sort_by,
|
||||
order,
|
||||
filters,
|
||||
});
|
||||
|
||||
if (filters.owner) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
@@ -389,28 +402,6 @@ class CatalogService implements OnModuleInit {
|
||||
return { data_asset: asset[0] };
|
||||
}
|
||||
|
||||
async updateCertificationStatus(data: {
|
||||
data_asset_id: string;
|
||||
body: IUpdateCertificationStatusRequest;
|
||||
metadata: Metadata;
|
||||
}) {
|
||||
const { body, data_asset_id, metadata } = data;
|
||||
|
||||
await lastValueFrom(
|
||||
this.catalogWriteService.UpdateDataAsset(
|
||||
{
|
||||
id: data_asset_id,
|
||||
changes: JSON.stringify({
|
||||
certification_status: body.certification_status,
|
||||
}),
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
|
||||
return { certification_status: body.certification_status };
|
||||
}
|
||||
|
||||
async updateOneDataAsset(data: {
|
||||
data_asset_id: string;
|
||||
customer_id: string;
|
||||
@@ -526,22 +517,6 @@ class CatalogService implements OnModuleInit {
|
||||
return response;
|
||||
}
|
||||
|
||||
async findSchemas(metadata: Metadata) {
|
||||
this.logger.info('CatalogService - findSchemas');
|
||||
|
||||
try {
|
||||
const response = await lastValueFrom(
|
||||
this.catalogReadService.GetSchemas({}, metadata),
|
||||
);
|
||||
|
||||
return response;
|
||||
|
||||
} catch (error) {
|
||||
this.logger.error('Error fetching schemas:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
@@ -759,64 +734,6 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async renameTableOnNimbus(
|
||||
nimbusUrl: string,
|
||||
nimbusId: number,
|
||||
changes: { table_name?: string; table_schema?: string; display_name?: string },
|
||||
): Promise<void> {
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
|
||||
this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
|
||||
await axios.patch(endpoint, changes);
|
||||
}
|
||||
|
||||
async renameColumnMetadataOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
|
||||
const { data: columns } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(columns) ? columns : [];
|
||||
|
||||
for (const column of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
|
||||
}
|
||||
|
||||
async renameDataPreviewOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
|
||||
const { data: previews } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(previews) ? previews : [];
|
||||
|
||||
for (const preview of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
|
||||
}
|
||||
|
||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||
const customer_name_raw = metadata.get('customer_name');
|
||||
|
||||
|
||||
@@ -1,10 +1,4 @@
|
||||
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
|
||||
import {
|
||||
IsEnum,
|
||||
IsNotEmpty,
|
||||
IsOptional,
|
||||
IsString,
|
||||
} from 'class-validator';
|
||||
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||
|
||||
export enum DataAssetShareType {
|
||||
@@ -12,12 +6,6 @@ export enum DataAssetShareType {
|
||||
public = 'public',
|
||||
private = 'private',
|
||||
}
|
||||
export enum CertificationStatus {
|
||||
draft = 'draft',
|
||||
in_review = 'in_review',
|
||||
approved = 'approved',
|
||||
deprecated = 'deprecated',
|
||||
}
|
||||
export enum OrderEnum {
|
||||
asc = 'asc',
|
||||
desc = 'desc',
|
||||
@@ -203,27 +191,6 @@ export class IData {
|
||||
day_opening: number;
|
||||
}
|
||||
|
||||
|
||||
export enum CustomPropertyType {
|
||||
TEXT = 'text',
|
||||
NUMBER = 'number',
|
||||
DATE = 'date',
|
||||
BOOLEAN = 'boolean',
|
||||
}
|
||||
|
||||
export class CustomPropertyDto {
|
||||
@ApiProperty()
|
||||
key: string;
|
||||
@ApiProperty()
|
||||
value: string;
|
||||
@ApiProperty({ enum: CustomPropertyType })
|
||||
type: CustomPropertyType;
|
||||
@ApiPropertyOptional()
|
||||
color?: string;
|
||||
@ApiPropertyOptional()
|
||||
emoji?: string;
|
||||
}
|
||||
|
||||
export class IUpdateDataRequest {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@@ -237,16 +204,7 @@ export class IUpdateDataRequest {
|
||||
share_type?: DataAssetShareType;
|
||||
@ApiPropertyOptional()
|
||||
docs?: string;
|
||||
@ApiPropertyOptional({ type: [CustomPropertyDto] })
|
||||
custom_properties?: CustomPropertyDto[];
|
||||
}
|
||||
|
||||
export class IUpdateCertificationStatusRequest {
|
||||
@ApiProperty({ enum: CertificationStatus })
|
||||
@IsEnum(CertificationStatus)
|
||||
certification_status: CertificationStatus;
|
||||
}
|
||||
|
||||
export class ICreateDataAsset implements CreateDataAssetRequest {
|
||||
@ApiProperty()
|
||||
display_name: string;
|
||||
@@ -400,4 +358,4 @@ export type CreateDataDocsDTO = {
|
||||
docs: string;
|
||||
asset_type: string;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -262,7 +262,6 @@ export class ShareService implements OnModuleInit {
|
||||
user_id: accessTokenPayload.user_id,
|
||||
username: accessTokenPayload.username,
|
||||
permissions: accessTokenPayload.permissions,
|
||||
roles: accessTokenPayload.roles,
|
||||
customer_id: accessTokenPayload.customer_id,
|
||||
customer_name: accessTokenPayload.customer_name,
|
||||
customer_tier: accessTokenPayload.customer_tier,
|
||||
|
||||
@@ -22,9 +22,6 @@ import {
|
||||
ConnectionTestListTablesRes,
|
||||
GetTableMetadataRes,
|
||||
GetTableMetadataReq,
|
||||
RefreshCatalogReq,
|
||||
RefreshCatalogRes,
|
||||
RefreshCatalogStatusReq,
|
||||
} from './dto/connection-test';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||
@@ -87,7 +84,7 @@ export class ConnectionTestController {
|
||||
});
|
||||
return this.connectionTestService.connectionTestListSchemas(
|
||||
body,
|
||||
user,
|
||||
user.customer_name,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -104,7 +101,7 @@ export class ConnectionTestController {
|
||||
});
|
||||
return this.connectionTestService.connectionTestListTables(
|
||||
body,
|
||||
user,
|
||||
user.customer_name,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -121,38 +118,7 @@ export class ConnectionTestController {
|
||||
});
|
||||
return this.connectionTestService.getTableMetadata(
|
||||
body,
|
||||
user,
|
||||
user.customer_name,
|
||||
);
|
||||
}
|
||||
|
||||
@Post('refresh-catalog')
|
||||
@ApiOkResponse({ type: RefreshCatalogRes })
|
||||
@HttpCode(HttpStatus.ACCEPTED)
|
||||
async refreshCatalog(
|
||||
@User() user: RequestUser,
|
||||
@Body(new ValidationPipe()) body: RefreshCatalogReq,
|
||||
) {
|
||||
this.logger.info('/connection-test/refresh-catalog', {
|
||||
user: user.user_id,
|
||||
customer: user.customer_name,
|
||||
connection: body.connection_id,
|
||||
});
|
||||
return this.connectionTestService.refreshCatalog(body, user);
|
||||
}
|
||||
|
||||
@Post('refresh-catalog/status')
|
||||
@ApiOkResponse({ type: RefreshCatalogRes })
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async refreshCatalogStatus(
|
||||
@User() user: RequestUser,
|
||||
@Body(new ValidationPipe()) body: RefreshCatalogStatusReq,
|
||||
) {
|
||||
this.logger.info('/connection-test/refresh-catalog/status', {
|
||||
user: user.user_id,
|
||||
customer: user.customer_name,
|
||||
connection: body.connection_id,
|
||||
session: body.session_id,
|
||||
});
|
||||
return this.connectionTestService.refreshCatalogStatus(body, user);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,17 +5,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { ConnectionTestClientConfiguration } from './connection-test-client.config';
|
||||
import { ConnectionModule } from '../connection/connection.module';
|
||||
import { ConnectionsApiModule } from '../connections-api/connections-api.module';
|
||||
import { PlatformApiModule } from '../platform-api/platform-api.module';
|
||||
const client = new ConnectionTestClientConfiguration();
|
||||
@Module({
|
||||
controllers: [ConnectionTestController],
|
||||
providers: [ConnectionTestService, DadosferaLogger],
|
||||
imports: [
|
||||
ClientsModule.register([client.providerOptions]),
|
||||
ConnectionModule,
|
||||
ConnectionsApiModule,
|
||||
PlatformApiModule,
|
||||
],
|
||||
imports: [ClientsModule.register([client.providerOptions]), ConnectionModule],
|
||||
})
|
||||
export class ConnectionTestModule {}
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
import { ConnectionTestService } from './connection-test.service';
|
||||
import { RequestUser } from 'src/decorators/user.decorator';
|
||||
|
||||
describe('ConnectionTestService catalog cache', () => {
|
||||
const user: RequestUser = {
|
||||
user_id: 'user-id',
|
||||
username: 'user@example.com',
|
||||
permissions: [],
|
||||
customer_id: 'customer-id',
|
||||
customer_name: 'customer-name',
|
||||
customer_tier: 'standard',
|
||||
access_token: 'token',
|
||||
customer_modules: [],
|
||||
roles: [],
|
||||
};
|
||||
const grpcClient = { getService: jest.fn().mockReturnValue({}) };
|
||||
const connectionsService = {};
|
||||
const connectionsApiService = { proxy: jest.fn() };
|
||||
const platformApiService = { proxy: jest.fn() };
|
||||
let service: ConnectionTestService;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
service = new ConnectionTestService(
|
||||
grpcClient as any,
|
||||
connectionsService as any,
|
||||
connectionsApiService as any,
|
||||
platformApiService as any,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps the existing schemas response contract', async () => {
|
||||
connectionsApiService.proxy.mockResolvedValue({
|
||||
schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }],
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.connectionTestListSchemas(
|
||||
{ connection_id: 'config-id', plugin: 'postgresql' },
|
||||
user,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
operation_result: true,
|
||||
schema_list: ['analytics', 'public'],
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps the existing tables response contract', async () => {
|
||||
connectionsApiService.proxy.mockResolvedValue({
|
||||
tables: [{ table_name: 'customers' }, { table_name: 'orders' }],
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.connectionTestListTables(
|
||||
{
|
||||
connection_id: 'config-id',
|
||||
plugin: 'postgresql',
|
||||
schema: 'public',
|
||||
},
|
||||
user,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
operation_result: true,
|
||||
table_list: ['customers', 'orders'],
|
||||
});
|
||||
});
|
||||
|
||||
it('maps cached columns to the existing table metadata contract', async () => {
|
||||
connectionsApiService.proxy.mockResolvedValue({
|
||||
columns: [
|
||||
{
|
||||
column_name: 'id',
|
||||
data_type: 'bigint',
|
||||
is_primary_key: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.getTableMetadata(
|
||||
{
|
||||
connection_id: 'config-id',
|
||||
plugin: 'postgresql',
|
||||
schema: 'public',
|
||||
table_list: ['customers'],
|
||||
},
|
||||
user,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
operation_result: true,
|
||||
tables_metadata: [
|
||||
{
|
||||
table_name: 'customers',
|
||||
columns: [
|
||||
{
|
||||
name: 'id',
|
||||
type: 'bigint',
|
||||
is_primary_key: true,
|
||||
},
|
||||
],
|
||||
references: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
|
||||
'GET',
|
||||
'/connection_catalog/config-id/schemas/public/tables/customers/columns',
|
||||
user,
|
||||
);
|
||||
});
|
||||
|
||||
it('submits a catalog refresh without holding the request open', async () => {
|
||||
platformApiService.proxy.mockResolvedValue({
|
||||
session_id: 'session-id',
|
||||
date: '20260731',
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.refreshCatalog(
|
||||
{ connection_id: 'config-id', plugin: 'postgresql' },
|
||||
user,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
operation_result: true,
|
||||
status: 'PENDING',
|
||||
session_id: 'session-id',
|
||||
date: '20260731',
|
||||
});
|
||||
|
||||
expect(platformApiService.proxy).toHaveBeenCalledWith(
|
||||
'POST',
|
||||
'/connection_test',
|
||||
user,
|
||||
{
|
||||
customer_id: user.customer_name,
|
||||
plugin: 'postgresql',
|
||||
task: {
|
||||
task_type: 'refresh_catalog',
|
||||
connection: {
|
||||
provider: 'connection_manager',
|
||||
config_id: 'config-id',
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps polling without changing the catalog pointer while pending', async () => {
|
||||
platformApiService.proxy.mockResolvedValue({ status: 'PENDING' });
|
||||
|
||||
await expect(
|
||||
service.refreshCatalogStatus(
|
||||
{
|
||||
connection_id: 'config-id',
|
||||
plugin: 'postgresql',
|
||||
session_id: 'session-id',
|
||||
date: '20260731',
|
||||
},
|
||||
user,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
operation_result: false,
|
||||
status: 'PENDING',
|
||||
session_id: 'session-id',
|
||||
date: '20260731',
|
||||
});
|
||||
|
||||
expect(connectionsApiService.proxy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('publishes the catalog pointer after the refresh finishes', async () => {
|
||||
platformApiService.proxy.mockResolvedValue({ status: 'DONE' });
|
||||
connectionsApiService.proxy.mockResolvedValue({
|
||||
last_catalog_refresh_status: 'SUCCESS',
|
||||
});
|
||||
|
||||
await expect(
|
||||
service.refreshCatalogStatus(
|
||||
{
|
||||
connection_id: 'config/id',
|
||||
plugin: 'postgresql',
|
||||
session_id: 'session-id',
|
||||
date: '20260731',
|
||||
},
|
||||
user,
|
||||
),
|
||||
).resolves.toEqual({
|
||||
operation_result: true,
|
||||
status: 'DONE',
|
||||
session_id: 'session-id',
|
||||
date: '20260731',
|
||||
});
|
||||
|
||||
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
|
||||
'PUT',
|
||||
'/connection_config/config%2Fid/catalog_metadata',
|
||||
user,
|
||||
{
|
||||
last_catalog_refresh_status: 'SUCCESS',
|
||||
last_catalog_connection_test_date: '20260731',
|
||||
last_catalog_connection_test_session_id: 'session-id',
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { ConnectionTest } from '@dadosfera/protospack-v2';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
@@ -13,9 +13,6 @@ import {
|
||||
ConnectionTestPingRes,
|
||||
GetTableMetadataReq,
|
||||
GetTableMetadataRes,
|
||||
RefreshCatalogReq,
|
||||
RefreshCatalogRes,
|
||||
RefreshCatalogStatusReq,
|
||||
} from './dto/connection-test';
|
||||
import { ConnectionClientService } from '../connection/client.service';
|
||||
import {
|
||||
@@ -24,8 +21,6 @@ import {
|
||||
} from '../connection/dtos/connection';
|
||||
import { RequestUser } from 'src/decorators/user.decorator';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { ConnectionsApiService } from '../connections-api/connections-api.service';
|
||||
import { PlatformApiService } from '../platform-api/platform-api.service';
|
||||
|
||||
@Injectable()
|
||||
export class ConnectionTestService {
|
||||
@@ -33,8 +28,6 @@ export class ConnectionTestService {
|
||||
constructor(
|
||||
@Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
|
||||
private connectionsService: ConnectionClientService,
|
||||
private connectionsApiService: ConnectionsApiService,
|
||||
private platformApiService: PlatformApiService,
|
||||
) {
|
||||
this.connectionTestReadClient =
|
||||
grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
|
||||
@@ -154,137 +147,45 @@ export class ConnectionTestService {
|
||||
}
|
||||
async connectionTestListSchemas(
|
||||
body: ConnectionTestListSchemasReq,
|
||||
user: RequestUser,
|
||||
customer_name: string,
|
||||
): Promise<ConnectionTestListSchemasRes> {
|
||||
const result = await this.connectionsApiService.proxy(
|
||||
'GET',
|
||||
`/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`,
|
||||
user,
|
||||
const { connection_id, plugin } = body;
|
||||
return lastValueFrom(
|
||||
this.connectionTestReadClient.ListSchemas({
|
||||
connection_id,
|
||||
customer_name,
|
||||
plugin,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
operation_result: true,
|
||||
schema_list: result.schemas.map((schema) => schema.schema_name),
|
||||
};
|
||||
}
|
||||
|
||||
async connectionTestListTables(
|
||||
body: ConnectionTestListTablesReq,
|
||||
user: RequestUser,
|
||||
customer_name: string,
|
||||
): Promise<ConnectionTestListTablesRes> {
|
||||
const result = await this.connectionsApiService.proxy(
|
||||
'GET',
|
||||
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
|
||||
`/schemas/${encodeURIComponent(body.schema)}/tables`,
|
||||
user,
|
||||
const { connection_id, plugin, schema } = body;
|
||||
return lastValueFrom(
|
||||
this.connectionTestReadClient.ListTables({
|
||||
connection_id,
|
||||
customer_name,
|
||||
plugin,
|
||||
schema,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
operation_result: true,
|
||||
table_list: result.tables.map((table) => table.table_name),
|
||||
};
|
||||
}
|
||||
|
||||
async getTableMetadata(
|
||||
body: GetTableMetadataReq,
|
||||
user: RequestUser,
|
||||
customer_name: string,
|
||||
): Promise<GetTableMetadataRes> {
|
||||
const tables_metadata = await Promise.all(
|
||||
body.table_list.map(async (table_name) => {
|
||||
const result = await this.connectionsApiService.proxy(
|
||||
'GET',
|
||||
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
|
||||
`/schemas/${encodeURIComponent(body.schema)}` +
|
||||
`/tables/${encodeURIComponent(table_name)}/columns`,
|
||||
user,
|
||||
);
|
||||
return {
|
||||
table_name,
|
||||
columns: result.columns.map((column) => ({
|
||||
name: column.column_name,
|
||||
type: column.data_type,
|
||||
is_primary_key: column.is_primary_key,
|
||||
})),
|
||||
references: [],
|
||||
};
|
||||
const { schema, plugin, table_list, connection_id } = body;
|
||||
return lastValueFrom(
|
||||
this.connectionTestReadClient.GetTableMetadata({
|
||||
connection_id,
|
||||
customer_name,
|
||||
plugin,
|
||||
schema,
|
||||
table_list,
|
||||
}),
|
||||
);
|
||||
return { operation_result: true, tables_metadata };
|
||||
}
|
||||
|
||||
async refreshCatalog(
|
||||
body: RefreshCatalogReq,
|
||||
user: RequestUser,
|
||||
): Promise<RefreshCatalogRes> {
|
||||
const task = await this.platformApiService.proxy(
|
||||
'POST',
|
||||
'/connection_test',
|
||||
user,
|
||||
{
|
||||
customer_id: user.customer_name,
|
||||
plugin: body.plugin,
|
||||
task: {
|
||||
task_type: 'refresh_catalog',
|
||||
connection: {
|
||||
provider: 'connection_manager',
|
||||
config_id: body.connection_id,
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (!task.session_id || !task.date) {
|
||||
throw new HttpException(
|
||||
'Platform API did not return a catalog refresh task identifier',
|
||||
HttpStatus.BAD_GATEWAY,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
operation_result: true,
|
||||
status: 'PENDING',
|
||||
session_id: task.session_id,
|
||||
date: task.date,
|
||||
};
|
||||
}
|
||||
|
||||
async refreshCatalogStatus(
|
||||
body: RefreshCatalogStatusReq,
|
||||
user: RequestUser,
|
||||
): Promise<RefreshCatalogRes> {
|
||||
const result = await this.platformApiService.proxy(
|
||||
'POST',
|
||||
'/connection_test/status',
|
||||
user,
|
||||
{
|
||||
session_id: body.session_id,
|
||||
date: body.date,
|
||||
},
|
||||
);
|
||||
|
||||
if (result.status === 'DONE') {
|
||||
await this.connectionsApiService.proxy(
|
||||
'PUT',
|
||||
`/connection_config/${encodeURIComponent(
|
||||
body.connection_id,
|
||||
)}/catalog_metadata`,
|
||||
user,
|
||||
{
|
||||
last_catalog_refresh_status: 'SUCCESS',
|
||||
last_catalog_connection_test_date: body.date,
|
||||
last_catalog_connection_test_session_id: body.session_id,
|
||||
},
|
||||
);
|
||||
} else if (result.status === 'ERROR' || result.status === 'EXPIRED') {
|
||||
throw new HttpException(
|
||||
`Catalog refresh finished with status ${result.status}`,
|
||||
HttpStatus.BAD_GATEWAY,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
operation_result: result.status === 'DONE',
|
||||
status: result.status,
|
||||
session_id: body.session_id,
|
||||
date: body.date,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||
import { IsIn, IsString, IsOptional } from 'class-validator';
|
||||
import { IsString, IsOptional } from 'class-validator';
|
||||
import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
|
||||
import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
|
||||
export class ColumnDto {
|
||||
@@ -7,8 +7,6 @@ export class ColumnDto {
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
@ApiProperty()
|
||||
is_primary_key: boolean;
|
||||
}
|
||||
export class TableMetadataDto {
|
||||
@ApiProperty()
|
||||
@@ -133,37 +131,3 @@ export class GetTableMetadataRes {
|
||||
@ApiProperty({ type: [TableMetadataDto] })
|
||||
tables_metadata: TableMetadataDto[];
|
||||
}
|
||||
|
||||
export class RefreshCatalogReq {
|
||||
@ApiProperty()
|
||||
@IsString()
|
||||
connection_id: string;
|
||||
|
||||
@ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] })
|
||||
@IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver'])
|
||||
plugin: string;
|
||||
}
|
||||
|
||||
export class RefreshCatalogStatusReq extends RefreshCatalogReq {
|
||||
@ApiProperty()
|
||||
@IsString()
|
||||
session_id: string;
|
||||
|
||||
@ApiProperty()
|
||||
@IsString()
|
||||
date: string;
|
||||
}
|
||||
|
||||
export class RefreshCatalogRes {
|
||||
@ApiProperty()
|
||||
operation_result: boolean;
|
||||
|
||||
@ApiProperty()
|
||||
status: string;
|
||||
|
||||
@ApiProperty()
|
||||
session_id: string;
|
||||
|
||||
@ApiProperty()
|
||||
date: string;
|
||||
}
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
export const CONNECTIONS_API_CONFIG = {
|
||||
getUrl: (): string => {
|
||||
const url = process.env.CONNECTIONS_API_URL;
|
||||
if (!url) {
|
||||
throw new Error('CONNECTIONS_API_URL environment variable is not set');
|
||||
}
|
||||
return url;
|
||||
},
|
||||
region: process.env.AWS_REGION || 'us-east-1',
|
||||
timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -1,10 +0,0 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { ConnectionsApiService } from './connections-api.service';
|
||||
|
||||
@Module({
|
||||
providers: [ConnectionsApiService, DadosferaLogger],
|
||||
exports: [ConnectionsApiService],
|
||||
})
|
||||
export class ConnectionsApiModule {}
|
||||
@@ -9,12 +9,12 @@ import {
|
||||
} from '@nestjs/common';
|
||||
|
||||
import { firstValueFrom, lastValueFrom } from 'rxjs';
|
||||
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||
import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
import { CustomerLinksConfig } from './dtos/customers';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import {
|
||||
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
|
||||
)
|
||||
}
|
||||
|
||||
async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
|
||||
async getLinks(customerId: string) {
|
||||
try {
|
||||
const result = await lastValueFrom(
|
||||
this.customerService.CustomerGetLinks({ customerId }),
|
||||
this.customerService.CustomerFindOneById({ id: customerId }),
|
||||
);
|
||||
return (result.links as CustomerLinksConfig) || null;
|
||||
return result.customer?.links || [];
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
|
||||
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async setLinks(customerId: string, links: CustomerLinksConfig) {
|
||||
async setLinks(customerId: string, links: Link[]) {
|
||||
if (!customerId || !links) {
|
||||
throw new HttpException(null, HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
try {
|
||||
return await firstValueFrom(
|
||||
this.customerService.CustomerSetLinks({
|
||||
customerId,
|
||||
links: links as CustomerSetLinksRequest['links'],
|
||||
}),
|
||||
this.customerService.CustomerUpdate({
|
||||
id: customerId,
|
||||
links,
|
||||
} as CustomerUpdateRequest),
|
||||
);
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
|
||||
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class CustomerLinkItem {
|
||||
export class CustomerLink implements Link {
|
||||
@ApiProperty()
|
||||
href: string;
|
||||
@ApiProperty()
|
||||
@@ -8,59 +9,15 @@ export class CustomerLinkItem {
|
||||
@ApiProperty()
|
||||
description: string;
|
||||
@ApiPropertyOptional()
|
||||
iconSrc?: string;
|
||||
iconSrc: string;
|
||||
}
|
||||
|
||||
export class CustomerSidebarLinkItem {
|
||||
@ApiProperty()
|
||||
type: 'link';
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiProperty()
|
||||
link: string;
|
||||
@ApiPropertyOptional()
|
||||
icon?: string;
|
||||
}
|
||||
|
||||
export class CustomerSidebarMenuItem {
|
||||
@ApiProperty()
|
||||
type: 'menu';
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiPropertyOptional()
|
||||
icon?: string;
|
||||
@ApiProperty({ type: [CustomerSidebarLinkItem] })
|
||||
items: CustomerSidebarLinkItem[];
|
||||
}
|
||||
|
||||
export class CustomerSidebarSection {
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiProperty({
|
||||
type: 'array',
|
||||
items: {
|
||||
oneOf: [
|
||||
{ $ref: '#/components/schemas/CustomerSidebarMenuItem' },
|
||||
{ $ref: '#/components/schemas/CustomerSidebarLinkItem' },
|
||||
],
|
||||
},
|
||||
})
|
||||
items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
|
||||
}
|
||||
|
||||
export class CustomerLinksConfig {
|
||||
@ApiPropertyOptional({ type: [CustomerLinkItem] })
|
||||
home?: CustomerLinkItem[];
|
||||
@ApiPropertyOptional({ type: [CustomerSidebarSection] })
|
||||
sidebar?: CustomerSidebarSection[];
|
||||
}
|
||||
|
||||
export class CustomerLinkRequest {
|
||||
@ApiProperty({ type: CustomerLinksConfig })
|
||||
links: CustomerLinksConfig;
|
||||
@ApiProperty({ type: [CustomerLink] })
|
||||
links: CustomerLink[];
|
||||
}
|
||||
|
||||
export class CustomerLinksResponse {
|
||||
@ApiPropertyOptional({ type: CustomerLinksConfig })
|
||||
links?: CustomerLinksConfig;
|
||||
}
|
||||
@ApiProperty({ type: [CustomerLink] })
|
||||
links: CustomerLink[];
|
||||
}
|
||||
|
||||
|
||||
@@ -11,19 +11,17 @@ export class TableColumns {
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
columns: string[];
|
||||
@ApiPropertyOptional({ type: [Column] })
|
||||
@ApiProperty()
|
||||
references: Column[];
|
||||
@ApiProperty()
|
||||
identifier_columns: string[];
|
||||
@ApiProperty()
|
||||
destination: Record<'raw' | 'qualify', {
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
}> | null;
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
@ApiPropertyOptional({ type: [String] })
|
||||
identifier_columns?: string[];
|
||||
@ApiPropertyOptional({ type: Column })
|
||||
reference_column?: Column;
|
||||
}
|
||||
export class AvailableEntity {
|
||||
@ApiProperty()
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
export interface Info {
|
||||
user_id: string;
|
||||
customer_id: string;
|
||||
customer: string;
|
||||
}
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
interface Values {
|
||||
jdbc_user: string;
|
||||
|
||||
@@ -99,7 +99,6 @@ export class InputsController {
|
||||
customer: info.customer,
|
||||
});
|
||||
|
||||
this.logger.info(JSON.stringify(body))
|
||||
const response = await this.inputService.create({ body, info });
|
||||
|
||||
return response;
|
||||
|
||||
@@ -17,13 +17,10 @@ import {
|
||||
InputCreateGenericRequest,
|
||||
InputCreateS3Request,
|
||||
InputNewCreateRequest,
|
||||
InputUpdateResponse,
|
||||
RollbackInputRequest,
|
||||
TestConnectionRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { CreateInputReq } from './dtos/input.model';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
|
||||
@Injectable()
|
||||
|
||||
@@ -74,10 +71,10 @@ export class InputsService {
|
||||
objectCamelToSnake(createInputResponse);
|
||||
return createInputResponse;
|
||||
},
|
||||
update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
|
||||
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
|
||||
update: async (updateInputDTO: UpdateInputRequest) => {
|
||||
this.logger.info('InputClientService - Update');
|
||||
const updateInputResponse = await lastValueFrom(
|
||||
this.inputWriteService.InputUpdate(updateInputDTO, metadata),
|
||||
this.inputWriteService.InputUpdate(updateInputDTO),
|
||||
);
|
||||
|
||||
return updateInputResponse;
|
||||
@@ -167,11 +164,6 @@ export class InputsService {
|
||||
const inputCreateGenericRequest: InputCreateGenericRequest = {
|
||||
input: {
|
||||
...body,
|
||||
tables: (body.tables || []).map((table) => ({
|
||||
...table,
|
||||
identifier_columns: table.identifier_columns || [],
|
||||
reference_column: table.reference_column || table.references?.[0],
|
||||
})),
|
||||
},
|
||||
info,
|
||||
};
|
||||
@@ -207,47 +199,24 @@ export class InputsService {
|
||||
return findOneInputResponse;
|
||||
}
|
||||
|
||||
async update(id: string, data, info: Info, metadata?: Metadata) {
|
||||
async update(id: string, data, info: Info) {
|
||||
// this.validateCron({ ...data, info });
|
||||
try {
|
||||
const {
|
||||
tablesUpdate,
|
||||
dataAssetUpdate,
|
||||
input
|
||||
} = await this.OLD_inputClient.update({
|
||||
const updateInputResponse: any = await this.OLD_inputClient.update({
|
||||
id,
|
||||
...data,
|
||||
info,
|
||||
}, metadata);
|
||||
...data,
|
||||
});
|
||||
|
||||
const updateInputResponse = this.adjustInputPayload(
|
||||
input,
|
||||
updateInputResponse.input = this.adjustInputPayload(
|
||||
updateInputResponse?.input,
|
||||
);
|
||||
return {
|
||||
input: updateInputResponse,
|
||||
tablesUpdate,
|
||||
dataAssetUpdate
|
||||
};
|
||||
return updateInputResponse;
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
async rollbackUpdate(
|
||||
data: RollbackInputRequest
|
||||
) {
|
||||
this.logger.info('PipelinesClientService - rollbackUpdate');
|
||||
this.logger.info('Rolling back input update with data: ' + JSON.stringify(data));
|
||||
const updatePipelineResponse = await lastValueFrom(
|
||||
this.inputWriteService.RollbackInputUpdate(
|
||||
data
|
||||
),
|
||||
);
|
||||
this.logger.info('Done');
|
||||
|
||||
return updatePipelineResponse;
|
||||
}
|
||||
|
||||
async remove(idRequest: IIdRequest) {
|
||||
return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
|
||||
}
|
||||
@@ -289,12 +258,4 @@ export class InputsService {
|
||||
};
|
||||
return formatedPayload;
|
||||
}
|
||||
|
||||
async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
|
||||
return lastValueFrom(this.inputWriteService.MarkTableDeleted(data));
|
||||
}
|
||||
|
||||
async unmarkTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
|
||||
return lastValueFrom((this.inputWriteService as any).UnmarkTableDeleted(data));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import {
|
||||
PipelineServicesNames,
|
||||
PipelinesServiceInterface,
|
||||
} from '@dadosfera/protospack';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
|
||||
import { IIdRequest } from './interfaces';
|
||||
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
|
||||
export class PipelinesClientService implements OnModuleInit {
|
||||
private pipelineService: PipelinesServiceInterface;
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
@Inject(PipelinesClientConfiguration.name)
|
||||
private readonly grpcClient: ClientGrpc,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
this.pipelineService =
|
||||
this.grpcClient.getService<PipelinesServiceInterface>(
|
||||
PipelineServicesNames.PipelineService,
|
||||
);
|
||||
}
|
||||
|
||||
async getPipelineStatus(data) {
|
||||
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
||||
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineService.getPipelineStatus(data),
|
||||
)
|
||||
.then((res) => {
|
||||
const statusArray =
|
||||
res.status?.sort((a, b) => {
|
||||
if (a.id < b.id) {
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}) || [];
|
||||
return { status: statusArray };
|
||||
})
|
||||
.catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
this.logger.info('Done');
|
||||
|
||||
return statusPipelineResponse;
|
||||
}
|
||||
|
||||
async runPipeline({ id, info }: IIdRequest) {
|
||||
this.logger.info('PipelinesClientService - RunPipeline');
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineService.triggerPipeline({ id, info }),
|
||||
).catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
|
||||
if (statusPipelineResponse.status == false) {
|
||||
throw new ConflictException(
|
||||
'This pipeline is not ready yet to execute, Try again later!',
|
||||
);
|
||||
}
|
||||
|
||||
this.logger.info('Done');
|
||||
return statusPipelineResponse;
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export interface ICreatePipelineDto {
|
||||
input: IdRequest;
|
||||
transformations: IdRequest[];
|
||||
output: IdRequest;
|
||||
tags: string[];
|
||||
name: string;
|
||||
description: string;
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface IdRequest {
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface IIdRequest {
|
||||
id: string;
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface IUpdatePipelineRequest {
|
||||
input: IdRequest;
|
||||
transformations: IdRequest[];
|
||||
output: IdRequest;
|
||||
tags: string[];
|
||||
name: string;
|
||||
description: string;
|
||||
id: string;
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface IGetPipelineLogsRequest {
|
||||
id: string;
|
||||
details: string;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import {
|
||||
ClientsProviderAsyncOptions,
|
||||
GrpcOptions,
|
||||
Transport,
|
||||
} from '@nestjs/microservices';
|
||||
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
|
||||
import { credentials } from '@grpc/grpc-js';
|
||||
|
||||
const isLocalConnection =
|
||||
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
||||
process.env.PIFACTORY_URL.includes('0.0.0.0');
|
||||
|
||||
export class PipelinesClientConfiguration {
|
||||
public name = 'PipelinesClientConfiguration';
|
||||
private config: GrpcOptions = {
|
||||
transport: Transport.GRPC,
|
||||
options: {
|
||||
url: process.env.PIFACTORY_URL,
|
||||
package: PipelinePackages,
|
||||
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
||||
protoPath: PipelineProtoFilePath,
|
||||
loader: {
|
||||
keepCase: true,
|
||||
enums: String,
|
||||
defaults: false,
|
||||
},
|
||||
},
|
||||
};
|
||||
providerOptions: ClientsProviderAsyncOptions = {
|
||||
name: this.name,
|
||||
...this.config,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
|
||||
import { ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||
import {
|
||||
AuthenticateCondition,
|
||||
Authenticated,
|
||||
RequireSomePermission,
|
||||
} from 'src/decorators/authentication.decorator';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { PipelinesService } from './pipelines.service';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||
|
||||
@ApiInternalOnlyController()
|
||||
@ApiTags('Pipelines')
|
||||
@Controller('pipelines')
|
||||
@Authenticated()
|
||||
export class PipelinesController {
|
||||
logger: DadosferaLogger;
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private pipelineService: PipelinesService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Post('start/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
@ApiOperation({
|
||||
deprecated: true,
|
||||
description:
|
||||
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
|
||||
})
|
||||
async activate(@Param('id') id: string, @Body() body) {
|
||||
const { info } = body;
|
||||
|
||||
this.logger.info(
|
||||
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
|
||||
{
|
||||
user: body.info.user_id,
|
||||
customer: body.info.customer,
|
||||
},
|
||||
);
|
||||
|
||||
const response = await this.pipelineService.runPipeline({ id, info });
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
@Get(':id/status')
|
||||
@ApiOperation({
|
||||
deprecated: true,
|
||||
description:
|
||||
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
|
||||
})
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||
body.id = id;
|
||||
|
||||
this.logger.info(
|
||||
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
|
||||
{
|
||||
user: body.info.user_id,
|
||||
customer: body.info.customer,
|
||||
},
|
||||
);
|
||||
|
||||
const response = await this.pipelineService.getPipelineStatus(body);
|
||||
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { PipelinesController } from './pipelines.controller';
|
||||
import { PipelinesService } from './pipelines.service';
|
||||
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
import { PipelinesClientService } from './client.service';
|
||||
|
||||
const client = new PipelinesClientConfiguration();
|
||||
|
||||
@Module({
|
||||
imports: [ClientsModule.register([client.providerOptions])],
|
||||
controllers: [PipelinesController],
|
||||
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
|
||||
exports: [PipelinesService],
|
||||
})
|
||||
export class PipelinesModule {}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
|
||||
import { PipelinesClientService } from './client.service';
|
||||
import { IIdRequest } from './interfaces';
|
||||
import { objectCamelToSnake } from 'src/utils/CaseConverter';
|
||||
|
||||
@Injectable()
|
||||
export class PipelinesService {
|
||||
constructor(private pipelineClient: PipelinesClientService) {}
|
||||
|
||||
async getPipelineStatus(data: IIdRequest) {
|
||||
try {
|
||||
const pipelineStatusResponse =
|
||||
await this.pipelineClient.getPipelineStatus(data);
|
||||
|
||||
return objectCamelToSnake(pipelineStatusResponse);
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
async runPipeline({ id, info }: IIdRequest) {
|
||||
try {
|
||||
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
|
||||
id,
|
||||
info,
|
||||
});
|
||||
|
||||
return objectCamelToSnake(triggerPipelineResponse);
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export class PipelineInputsDTO {
|
||||
@ApiProperty()
|
||||
@@ -61,12 +62,6 @@ export interface IIdRequest {
|
||||
info: Info;
|
||||
}
|
||||
|
||||
export interface Info {
|
||||
user_id: string;
|
||||
customer_id: string;
|
||||
customer: string;
|
||||
}
|
||||
|
||||
export interface IUpdatePipelineRequest {
|
||||
input: IdRequest;
|
||||
transformations: IdRequest[];
|
||||
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
Patch,
|
||||
HttpException,
|
||||
BadRequestException,
|
||||
UseGuards,
|
||||
CacheTTL,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
ApiTags,
|
||||
} from '@nestjs/swagger';
|
||||
import {
|
||||
AuthenticateCondition,
|
||||
RequireAllPermissions,
|
||||
RequireSomePermission,
|
||||
} from 'src/decorators/authentication.decorator';
|
||||
@@ -34,6 +35,7 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
|
||||
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
|
||||
import {
|
||||
ICompleteUploadCSVFile,
|
||||
ICreatePipelineCSVFile,
|
||||
@@ -47,11 +49,9 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
|
||||
import { TableColumns } from '../inputs/dtos/input.model';
|
||||
import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
|
||||
|
||||
type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any };
|
||||
type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
|
||||
|
||||
@ApiTags('PipelinesV2')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@@ -63,6 +63,7 @@ export class PipelinesController {
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private pipelinesClientService: PipelinesService,
|
||||
private oldPipelinesService: OldPipelineService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
@@ -190,7 +191,6 @@ export class PipelinesController {
|
||||
@Get(':id/status')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||
|
||||
body.id = id;
|
||||
|
||||
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
|
||||
@@ -198,7 +198,7 @@ export class PipelinesController {
|
||||
customer: body.info.customer,
|
||||
});
|
||||
|
||||
const response = await this.pipelinesClientService.getPipelineStatus(body);
|
||||
const response = await this.oldPipelinesService.getPipelineStatus(body);
|
||||
|
||||
return response;
|
||||
}
|
||||
@@ -222,50 +222,28 @@ export class PipelinesController {
|
||||
language,
|
||||
});
|
||||
|
||||
const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata);
|
||||
|
||||
const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables);
|
||||
const input_id = parsed.input_id;
|
||||
const tables: PipelineTable[] = parsed.tables ?? [];
|
||||
|
||||
Object.assign(pipelineRes.pipeline, {
|
||||
transformations: pipelineRes.pipeline.transformations
|
||||
? JSON.parse(pipelineRes.pipeline.transformations)
|
||||
: [],
|
||||
config: {
|
||||
cron: pipelineRes.pipeline.config.cron,
|
||||
tables,
|
||||
input_id,
|
||||
},
|
||||
properties: pipelineRes.pipeline.properties
|
||||
? JSON.parse(pipelineRes.pipeline.properties)
|
||||
: {},
|
||||
});
|
||||
|
||||
return pipelineRes;
|
||||
}
|
||||
|
||||
@Get("/:id/data-assets")
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET
|
||||
)
|
||||
async findAllDataAssetByPipeline(
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
@Query('object') object: string
|
||||
) {
|
||||
const payload = {
|
||||
pipeline: id,
|
||||
object: object,
|
||||
};
|
||||
|
||||
this.logger.info(`GET pipelinesV2/:id/data-assets` + JSON.stringify(payload));
|
||||
|
||||
const result =
|
||||
await this.pipelinesClientService.findAllDataAssetByPipeline(payload, user);
|
||||
const result = await this.pipelinesClientService
|
||||
.findOne({ id }, metadata)
|
||||
.then((res) => {
|
||||
//{pipeline:{tables: {tables: [], input_id: ''}}}
|
||||
let tables = JSON.parse(res.pipeline.config.tables);
|
||||
const input_id = tables?.input_id;
|
||||
if (tables?.tables) tables = tables.tables;
|
||||
Object.assign(res.pipeline, {
|
||||
transformations: res.pipeline.transformations
|
||||
? JSON.parse(res.pipeline.transformations)
|
||||
: [],
|
||||
config: {
|
||||
cron: res.pipeline.config.cron,
|
||||
tables,
|
||||
input_id
|
||||
},
|
||||
properties: res.pipeline.properties
|
||||
? JSON.parse(res.pipeline.properties)
|
||||
: {},
|
||||
});
|
||||
return res;
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -307,7 +285,6 @@ export class PipelinesController {
|
||||
|
||||
@Patch('/:pipelineId/inputs/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
@UseGuards(PipelineExecutionGuard)
|
||||
async updatePipelineInput(
|
||||
@Language() language: LanguageEnum,
|
||||
@Body() pipelineInputDTO: UpdatePlatformInputRequest,
|
||||
@@ -317,14 +294,20 @@ export class PipelinesController {
|
||||
) {
|
||||
this.logger.info('PipelinesController - update', { user });
|
||||
|
||||
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const info: Info = {
|
||||
user_id: user.user_id,
|
||||
customer: user.customer_name,
|
||||
customer_id: user.customer_id,
|
||||
pipeline_id: pipelineId
|
||||
};
|
||||
|
||||
const metadata = PackTheMetadata(user);
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
language,
|
||||
});
|
||||
|
||||
const response = await this.pipelinesClientService.updatePipelineInput(
|
||||
pipelineId,
|
||||
@@ -358,21 +341,6 @@ export class PipelinesController {
|
||||
return response;
|
||||
}
|
||||
|
||||
@Patch('/:id/upgrade')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
async upgradeConnector(
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser
|
||||
) {
|
||||
this.logger.info('PipelinesController - upgrade connector');
|
||||
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
await this.pipelinesClientService.upgrade(id, metadata);
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@ApiNoContentResponse()
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@@ -502,7 +470,7 @@ export class PipelinesController {
|
||||
},
|
||||
);
|
||||
|
||||
const response = await this.pipelinesClientService.runPipeline({ id, info });
|
||||
const response = await this.oldPipelinesService.runPipeline({ id, info });
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
@@ -7,28 +7,25 @@ import { PipelinesService } from './pipelines.service';
|
||||
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
|
||||
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||
import { ConnectorModule } from '../connector/connector.module';
|
||||
import { InputsModule } from '../inputs/inputs.module';
|
||||
import { TransformationsModule } from '../transformations/transformations.module';
|
||||
import { PlatformApiModule } from '../platform-api/platform-api.module';
|
||||
import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
import { CatalogModule } from '../catalog/catalog.module';
|
||||
|
||||
const client = new PipelinesClientConfiguration();
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ClientsModule.register([client.providerOptions]),
|
||||
OldPipelineModule,
|
||||
ConnectorModule,
|
||||
InputsModule,
|
||||
TransformationsModule,
|
||||
PlatformApiModule,
|
||||
NimbusServicesModule,
|
||||
CatalogModule
|
||||
PlatformApiModule
|
||||
],
|
||||
controllers: [PipelinesController],
|
||||
providers: [PipelinesService, DadosferaLogger, NimbusService],
|
||||
providers: [PipelinesService, DadosferaLogger],
|
||||
exports: [PipelinesService],
|
||||
})
|
||||
export class PipelinesV2Module {}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/* eslint-disable no-async-promise-executor */
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Inject,
|
||||
@@ -18,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
|
||||
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { ConnectorClientService } from '../connector/client.service';
|
||||
@@ -30,15 +29,6 @@ import ErrorCodes from 'src/utils/errorCodes';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
import { PlatformApiService } from '../platform-api/platform-api.service';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { AxiosError } from 'axios';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { IDataAsset } from '../catalog/dtos';
|
||||
import { CatalogService } from '../catalog/catalog.service';
|
||||
|
||||
type RollbackPromise = () => Promise<any>;
|
||||
|
||||
export class PipelinesService implements OnModuleInit {
|
||||
logger: DadosferaLogger;
|
||||
@@ -52,9 +42,7 @@ export class PipelinesService implements OnModuleInit {
|
||||
private readonly connectorService: ConnectorClientService,
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly transformationsService: TransformationsService,
|
||||
private readonly platformAPI: PlatformApiService,
|
||||
private readonly nimbusService: NimbusService,
|
||||
private readonly catalogService: CatalogService
|
||||
private readonly platformAPI: PlatformApiService
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
@@ -177,15 +165,6 @@ export class PipelinesService implements OnModuleInit {
|
||||
return updatePipelineResponse;
|
||||
}
|
||||
|
||||
async upgrade(id: string, metadata: Metadata) {
|
||||
await lastValueFrom(
|
||||
this.pipelineWriteService.Upgrade(
|
||||
{ id },
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
|
||||
const { id, metadata, user } = data;
|
||||
const info = {
|
||||
@@ -369,316 +348,133 @@ export class PipelinesService implements OnModuleInit {
|
||||
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
|
||||
this.logger.info('InputClientService - Update');
|
||||
|
||||
const {
|
||||
input: oldInput
|
||||
} = await this.inputsService.findOne({
|
||||
id: inputId,
|
||||
info: info
|
||||
});
|
||||
|
||||
this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
|
||||
this.logger.info('Update Dynamo Reference');
|
||||
const pipelineIdFormat = pipelineId.split('-').join('_');
|
||||
const rollback: RollbackPromise[] = [];
|
||||
|
||||
const updateInputResponse = await this.inputsService.update(
|
||||
inputId,
|
||||
updateInputDTO,
|
||||
info,
|
||||
metadata
|
||||
);
|
||||
info
|
||||
)
|
||||
|
||||
const inputRollback = () => {
|
||||
this.logger.info("exec rollback to input: " + JSON.stringify(oldInput));
|
||||
return this.inputsService.rollbackUpdate(
|
||||
{
|
||||
id: inputId,
|
||||
dataAssetUpdate: updateInputResponse.dataAssetUpdate,
|
||||
tables: oldInput.tables,
|
||||
info
|
||||
}
|
||||
) as Promise<any>;
|
||||
}
|
||||
const requests = [];
|
||||
|
||||
rollback.push(inputRollback);
|
||||
|
||||
this.logger.info("Input Update Response: " + JSON.stringify(updateInputResponse))
|
||||
|
||||
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
|
||||
|
||||
if (user.customer_modules.includes('catalog')) {
|
||||
nimbusUpdates.forEach(update => {
|
||||
const nimbusRollback = () => {
|
||||
return this.nimbusService.renameTable(
|
||||
info.customer,
|
||||
update.database,
|
||||
{
|
||||
table_name: update.table_name,
|
||||
table_schema: update.table_schema
|
||||
},
|
||||
{
|
||||
table_name: update.old_table_name,
|
||||
table_schema: update.old_table_schema
|
||||
}
|
||||
);
|
||||
}
|
||||
rollback.push(nimbusRollback);
|
||||
});
|
||||
|
||||
try {
|
||||
await this.updateNimbus(info.customer, nimbusUpdates);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
if (error instanceof AxiosError) {
|
||||
this.logger.error(JSON.stringify(error.response.data));
|
||||
}
|
||||
await this.executeRenameRollback(rollback);
|
||||
|
||||
throw new Error("Error Nimbus updating tables");
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.updatePlatformJobs(
|
||||
pipelineIdFormat,
|
||||
updateInputResponse.input.type,
|
||||
updateInputDTO,
|
||||
user
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
await this.executeRenameRollback(rollback)
|
||||
throw new Error("Error Platform API updating jobs");
|
||||
}
|
||||
|
||||
return updateInputResponse;
|
||||
}
|
||||
|
||||
private async executeRenameRollback(request: RollbackPromise[]) {
|
||||
this.logger.info('rollback steps: ' + request.length)
|
||||
const result = await Promise.allSettled(request.map(func => func()));
|
||||
result.forEach(promise => {
|
||||
this.logger.info("Promise finish with status: " + promise.status)
|
||||
|
||||
if (promise.status === "rejected") {
|
||||
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}))
|
||||
}
|
||||
|
||||
if (promise.status === "fulfilled") {
|
||||
this.logger.info("success with: " + JSON.stringify(promise.value || {}))
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
private async updateNimbus(customer: string, changes: TableUpdate[]) {
|
||||
// throw new Error("teste error nimbus");
|
||||
this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
|
||||
if(!changes || changes.length === 0) return;
|
||||
|
||||
const requests = changes.map(change => {
|
||||
return this.nimbusService.renameTable(customer, change.database, {
|
||||
table_name: change.old_table_name,
|
||||
table_schema: change.old_table_schema
|
||||
}, {
|
||||
table_name: change.table_name,
|
||||
table_schema: change.table_schema
|
||||
});
|
||||
})
|
||||
|
||||
const values = await Promise.allSettled(requests);
|
||||
|
||||
const success = values.map(request => request.status === "fulfilled")
|
||||
|
||||
this.logger.info("Updates with succes: " + success.length);
|
||||
|
||||
values.forEach(promise => {
|
||||
this.logger.info("Promise finish with status: " + promise.status)
|
||||
|
||||
if (promise.status === "rejected") {
|
||||
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}));
|
||||
throw new Error(promise.reason );
|
||||
}
|
||||
|
||||
if (promise.status === "fulfilled") {
|
||||
this.logger.info("success with: " + JSON.stringify(promise.value || {}));
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
async updatePlatformJobs(pipelineId: string, pipelineType: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
|
||||
const jobsUpdated = [];
|
||||
this.logger.info('Dynamo Response', updateInputResponse);
|
||||
|
||||
for (const [index, table] of updateInputDTO.tables.entries()) {
|
||||
const jobUpdate = {
|
||||
job_id: `${pipelineId}_${index}`,
|
||||
}
|
||||
|
||||
if (table.type !== "incremental_with_qualify") {
|
||||
delete table.destinations?.qualify;
|
||||
}
|
||||
|
||||
if (table.memory) {
|
||||
jobUpdate["memory"] = {
|
||||
amount: table.memory * 1000
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Updating input reference for table: ' + table.name);
|
||||
let hasUpdateSyncMode = false;
|
||||
|
||||
const jobSyncMode = {}
|
||||
|
||||
const id = `${pipelineIdFormat}_${index}`;
|
||||
this.logger.info('Updating input reference for table', table.name);
|
||||
const body = {}
|
||||
if (table.columns) {
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['column_include_list'] = table.columns;
|
||||
body['column_include_list'] = table.columns;
|
||||
}
|
||||
|
||||
if (table.reference_column) {
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['incremental_column_name'] = table.reference_column.name;
|
||||
jobSyncMode['incremental_column_type'] = table.reference_column.type;
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
}
|
||||
|
||||
if (table.identifier_columns) {
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['primary_keys'] = table.identifier_columns;
|
||||
body['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
this.logger.info('Request body', body);
|
||||
const updateCollumns = this.platformAPI.proxy(
|
||||
'PATCH',
|
||||
`/jobs/${id}/input`,
|
||||
user,
|
||||
body
|
||||
)
|
||||
requests.push(updateCollumns);
|
||||
|
||||
if (table.memory) {
|
||||
this.logger.info('Updating memory allocation for table', table.name);
|
||||
const updateMemory = this.platformAPI.proxy(
|
||||
'PUT',
|
||||
`/jobs/${id}/memory`,
|
||||
user,
|
||||
{
|
||||
amount: table.memory
|
||||
}
|
||||
)
|
||||
requests.push(updateMemory);
|
||||
}
|
||||
|
||||
if (table.type) {
|
||||
hasUpdateSyncMode = true;
|
||||
|
||||
jobSyncMode['target_load_type'] = table.type;
|
||||
const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
|
||||
requests.push(updateSyncMode);
|
||||
}
|
||||
|
||||
if(hasUpdateSyncMode) {
|
||||
jobUpdate["sync_mode"] = jobSyncMode;
|
||||
}
|
||||
|
||||
if (Object.keys(table.destinations).length > 1) {
|
||||
let hasChanges = false
|
||||
const jobRenameTables = {
|
||||
raw: {},
|
||||
qualify: {}
|
||||
}
|
||||
|
||||
if (Object.keys(table.destinations.raw).length > 1) {
|
||||
hasChanges = true;
|
||||
jobRenameTables.raw = table.destinations.raw;
|
||||
}
|
||||
|
||||
if (Object.keys(table.destinations.qualify).length > 1) {
|
||||
hasChanges = true;
|
||||
jobRenameTables.qualify = table.destinations.qualify;
|
||||
}
|
||||
|
||||
if (hasChanges) {
|
||||
jobUpdate['rename_tables'] = jobRenameTables;
|
||||
}
|
||||
}
|
||||
|
||||
jobsUpdated.push(jobUpdate);
|
||||
}
|
||||
|
||||
this.logger.info('Request body:' + JSON.stringify({
|
||||
jobs_updated: jobsUpdated
|
||||
}));
|
||||
this.logger.info('Create Platform Request for each JOB');
|
||||
|
||||
const response = await this.platformAPI.proxy(
|
||||
'PUT',
|
||||
`/pipeline/${pipelineId}/jobs`,
|
||||
user,
|
||||
{
|
||||
job_updates: jobsUpdated
|
||||
}
|
||||
)
|
||||
this.logger.info('Platform api response: ' + JSON.stringify(response));
|
||||
}
|
||||
if (updateInputDTO.cron) {
|
||||
const crnUpdatedRequest = new Promise(async (resolve, reject) => {
|
||||
const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
|
||||
|
||||
async findAllDataAssetByPipeline(data: {
|
||||
pipeline: string,
|
||||
object?: string
|
||||
}, user: RequestUser) {
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
const isDataAdmin = user.permissions.includes(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
|
||||
);
|
||||
|
||||
let has_permission = false;
|
||||
|
||||
const {
|
||||
data_assets: resultString
|
||||
} = await lastValueFrom(
|
||||
this.pipelineReadService.FindAllDataAssetByPipeline(data, metadata)
|
||||
);
|
||||
|
||||
const result = JSON.parse(resultString) as any;
|
||||
const data_assets: IDataAsset[] = []
|
||||
result.forEach(data_asset => {
|
||||
if (data_asset?.owner === user.username) has_permission = true;
|
||||
|
||||
for (const role of user.roles) {
|
||||
if (data_asset.roles.includes(role)) has_permission = true;
|
||||
}
|
||||
|
||||
if (data_asset.users.includes(user.user_id)) has_permission = true;
|
||||
|
||||
if (isDataAdmin || has_permission) {
|
||||
delete data_asset.p_roles;
|
||||
delete data_asset.p_users;
|
||||
data_assets.push(data_asset as IDataAsset);
|
||||
}
|
||||
});
|
||||
|
||||
const assets = await this.catalogService.getAssetsUsersAndRoles(data_assets, user.customer_id);
|
||||
|
||||
return assets;
|
||||
}
|
||||
|
||||
async getPipelineStatus(data) {
|
||||
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
||||
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
|
||||
)
|
||||
.then((res) => {
|
||||
const statusArray =
|
||||
res.status?.sort((a, b) => {
|
||||
if (a.id < b.id) {
|
||||
return 1;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}) || [];
|
||||
return { status: statusArray };
|
||||
})
|
||||
.catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
if (response.error) {
|
||||
this.logger.error('Error updating pipeline cron', response.error);
|
||||
return reject(new ErrorBuilder(response.error));
|
||||
}
|
||||
this.logger.error('Pipeline cron updated successfully', response);
|
||||
return resolve(response);
|
||||
});
|
||||
this.logger.info('Done');
|
||||
|
||||
return statusPipelineResponse;
|
||||
}
|
||||
|
||||
async runPipeline({ id, info }: IIdRequest) {
|
||||
this.logger.info('PipelinesClientService - RunPipeline');
|
||||
const statusPipelineResponse = await lastValueFrom(
|
||||
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
|
||||
).catch((err) => {
|
||||
this.logger.error(err.message);
|
||||
throw new Error(err);
|
||||
});
|
||||
|
||||
if (statusPipelineResponse.status == false) {
|
||||
throw new ConflictException(
|
||||
'This pipeline is not ready yet to execute, Try again later!',
|
||||
);
|
||||
requests.push(crnUpdatedRequest);
|
||||
}
|
||||
|
||||
this.logger.info('Done');
|
||||
return statusPipelineResponse;
|
||||
this.logger.info('Executing all request for the platform api');
|
||||
|
||||
const results = await Promise.allSettled(requests);
|
||||
this.logger.info('Platform api response', results);
|
||||
|
||||
return updateInputResponse;
|
||||
|
||||
}
|
||||
|
||||
private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) {
|
||||
const body = {
|
||||
target_load_type: table.type
|
||||
}
|
||||
|
||||
if (table.type === 'incremental_with_qualify') {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
body['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
if (table.type === 'incremental') {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
}
|
||||
|
||||
this.logger.info('Updating pipeline sync mode', {
|
||||
pipelineId,
|
||||
body
|
||||
});
|
||||
|
||||
return this.platformAPI.proxy(
|
||||
"POST",
|
||||
`/jobs/jdbc/${pipelineId}/sync-mode`,
|
||||
user,
|
||||
body
|
||||
)
|
||||
}
|
||||
|
||||
private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
|
||||
try {
|
||||
const response = await this.platformAPI.proxy(
|
||||
'PATCH',
|
||||
`/pipeline/${pipelineId}`,
|
||||
user,
|
||||
{
|
||||
cron
|
||||
}
|
||||
);
|
||||
|
||||
return response
|
||||
} catch (error) {
|
||||
return {
|
||||
error: error.message
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,11 +10,8 @@ import {
|
||||
Query,
|
||||
Inject,
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
NotFoundException,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
|
||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import {
|
||||
@@ -28,11 +25,7 @@ import { ElasticsearchService } from '../../services/elasticsearch';
|
||||
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
|
||||
import { CustomersService } from '../customers/customers.service';
|
||||
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
||||
import { CatalogService } from '../catalog/catalog.service';
|
||||
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
||||
import { ValidationTableDTO } from './platform-api.dto';
|
||||
import { InputsService } from '../inputs/inputs.service';
|
||||
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
|
||||
|
||||
|
||||
type ValidateTablesDTO = {
|
||||
@@ -42,11 +35,6 @@ type ValidateTablesDTO = {
|
||||
}>
|
||||
}
|
||||
|
||||
type RenameTablesBody = {
|
||||
raw?: { table_name: string; table_schema: string };
|
||||
qualify?: { table_name: string; table_schema: string };
|
||||
}
|
||||
|
||||
@ApiTags('Platform API')
|
||||
@Controller('platform')
|
||||
export class PlatformApiController {
|
||||
@@ -57,8 +45,6 @@ export class PlatformApiController {
|
||||
private readonly elasticsearchService: ElasticsearchService,
|
||||
private readonly dynamoDBService: DynamoDBService,
|
||||
private readonly customersService: CustomersService,
|
||||
private readonly catalogService: CatalogService,
|
||||
private readonly inputsService: InputsService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
@@ -72,10 +58,6 @@ export class PlatformApiController {
|
||||
return id?.replace(/-/g, '_') || '';
|
||||
}
|
||||
|
||||
private decodePathParam(value: string): string {
|
||||
return value ? decodeURIComponent(value) : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Denormalize ID back to UUID format (replace _ with -).
|
||||
* Used when we receive a normalized ID but need the original UUID.
|
||||
@@ -94,23 +76,6 @@ export class PlatformApiController {
|
||||
return jobId?.replace(/-/g, '_') || '';
|
||||
}
|
||||
|
||||
private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise<any> {
|
||||
const connectorTypes = ['jdbc', 'singer', 's3'];
|
||||
for (const type of connectorTypes) {
|
||||
try {
|
||||
const job = await this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/jobs/${type}/${normalizedJobId}`,
|
||||
user,
|
||||
);
|
||||
return job;
|
||||
} catch (error) {
|
||||
// Continue to next connector type
|
||||
}
|
||||
}
|
||||
throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the pipeline ID (base UUID) from a job ID.
|
||||
* Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1").
|
||||
@@ -138,7 +103,7 @@ export class PlatformApiController {
|
||||
}
|
||||
|
||||
private readonly VALID_CONNECTORS = ['jdbc', 'singer', 's3'];
|
||||
private readonly MAX_MEMORY_MB = 12000; // 12GB maximum memory per pipelines/job
|
||||
private readonly MAX_MEMORY_MB = 12000; // 12GB maximum memory per pipeline/job
|
||||
|
||||
/**
|
||||
* Validate that connector is provided and is a valid type.
|
||||
@@ -409,9 +374,55 @@ export class PlatformApiController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync sync-mode changes to DynamoDB for JDBC connectors.
|
||||
* Always passes both target_load_type and incremental_column_name to ensure proper sync.
|
||||
*/
|
||||
private async syncJdbcSyncModeToDynamoDB(
|
||||
jobId: string,
|
||||
body: any,
|
||||
user: RequestUser,
|
||||
): Promise<void> {
|
||||
// JDBC sync mode uses target_load_type field
|
||||
const changes: any = {};
|
||||
|
||||
if ('target_load_type' in body) {
|
||||
changes.target_load_type = body.target_load_type;
|
||||
}
|
||||
|
||||
// Handle incremental_column_name:
|
||||
// - If provided in body, use that value
|
||||
// - If changing to full_load, explicitly clear it
|
||||
if ('incremental_column_name' in body) {
|
||||
changes.incremental_column_name = body.incremental_column_name;
|
||||
changes.incremental_column_type = body.incremental_column_type;
|
||||
} else if (body.target_load_type === 'full_load') {
|
||||
// Changing to full_load without specifying incremental_column - clear it
|
||||
changes.incremental_column_name = null;
|
||||
}
|
||||
|
||||
await this.syncJobInputToDynamoDB(jobId, changes, user, 'jdbc');
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync sync-mode changes to DynamoDB for Singer connectors.
|
||||
*/
|
||||
private async syncSingerSyncModeToDynamoDB(
|
||||
jobId: string,
|
||||
body: any,
|
||||
user: RequestUser,
|
||||
): Promise<void> {
|
||||
// Singer sync mode uses replication_method field
|
||||
// Map to DynamoDB type: FULL_TABLE -> full_load, INCREMENTAL -> incremental
|
||||
if ('replication_method' in body) {
|
||||
const type = body.replication_method === 'INCREMENTAL' ? 'incremental' : 'full_load';
|
||||
await this.syncJobInputToDynamoDB(jobId, { load_type: type }, user, 'singer');
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== PIPELINE ROUTES ====================
|
||||
|
||||
@Post('pipelines')
|
||||
@Post('pipeline')
|
||||
@ApiOperation({ summary: 'Create a new pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
async createPipeline(@Body() body: any, @User() user: RequestUser) {
|
||||
@@ -532,7 +543,7 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Get('pipelines/:pipelineId')
|
||||
@Get('pipeline/:pipelineId')
|
||||
@ApiOperation({ summary: 'Get pipeline by ID' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipeline(
|
||||
@@ -543,7 +554,7 @@ export class PlatformApiController {
|
||||
return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
|
||||
}
|
||||
|
||||
@Patch('pipelines/:pipelineId')
|
||||
@Patch('pipeline/:pipelineId')
|
||||
@ApiOperation({ summary: 'Update pipeline by ID' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updatePipeline(
|
||||
@@ -594,7 +605,7 @@ export class PlatformApiController {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Delete('pipelines/:pipelineId')
|
||||
@Delete('pipeline/:pipelineId')
|
||||
@ApiOperation({ summary: 'Delete pipeline by ID' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||
async deletePipeline(
|
||||
@@ -624,7 +635,7 @@ export class PlatformApiController {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Post('pipelines/execute')
|
||||
@Post('pipeline/execute')
|
||||
@ApiOperation({ summary: 'Execute a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async executePipeline(@Body() body: any, @User() user: RequestUser) {
|
||||
@@ -634,10 +645,10 @@ export class PlatformApiController {
|
||||
...body,
|
||||
customer_id: user.customer_name,
|
||||
};
|
||||
return this.platformApiService.proxy('POST', '/pipelines/execute', user, enrichedBody);
|
||||
return this.platformApiService.proxy('POST', '/pipeline/execute', user, enrichedBody);
|
||||
}
|
||||
|
||||
@Post('pipelines/pause')
|
||||
@Post('pipeline/pause')
|
||||
@ApiOperation({ summary: 'Pause a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async pausePipeline(@Body() body: any, @User() user: RequestUser) {
|
||||
@@ -650,7 +661,7 @@ export class PlatformApiController {
|
||||
return this.platformApiService.proxy('POST', '/pipeline/pause', user, enrichedBody);
|
||||
}
|
||||
|
||||
@Post('pipelines/unpause')
|
||||
@Post('pipeline/unpause')
|
||||
@ApiOperation({ summary: 'Unpause a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async unpausePipeline(@Body() body: any, @User() user: RequestUser) {
|
||||
@@ -663,7 +674,7 @@ export class PlatformApiController {
|
||||
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, enrichedBody);
|
||||
}
|
||||
|
||||
@Put('pipelines/:pipelineId/memory')
|
||||
@Put('pipeline/:pipelineId/memory')
|
||||
@ApiOperation({ summary: 'Update pipeline memory configuration' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updatePipelineMemory(
|
||||
@@ -686,7 +697,7 @@ export class PlatformApiController {
|
||||
|
||||
// ==================== PIPELINE METADATA ROUTES ====================
|
||||
|
||||
@Put('pipelines/:pipelineId/metadata')
|
||||
@Put('pipeline/:pipelineId/metadata')
|
||||
@ApiOperation({ summary: 'Update pipeline metadata' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updatePipelineMetadata(
|
||||
@@ -755,7 +766,7 @@ export class PlatformApiController {
|
||||
|
||||
// ==================== PIPELINE RUN ROUTES ====================
|
||||
|
||||
@Get('pipelines/:pipelineId/pipeline_run')
|
||||
@Get('pipeline/:pipelineId/pipeline_run')
|
||||
@ApiOperation({ summary: 'Get pipeline runs for a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineRuns(
|
||||
@@ -773,7 +784,7 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Get('pipelines/:pipelineId/pipeline_run/:runId')
|
||||
@Get('pipeline/:pipelineId/pipeline_run/:runId')
|
||||
@ApiOperation({ summary: 'Get specific pipeline run' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineRun(
|
||||
@@ -790,7 +801,7 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Get('pipelines/pipeline_run/:runId/logs')
|
||||
@Get('pipeline/pipeline_run/:runId/logs')
|
||||
@ApiOperation({ summary: 'Get pipeline run logs' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineRunLogs(
|
||||
@@ -808,68 +819,6 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Post('pipelines/:pipelineId/pipeline_run/:runId/cancel')
|
||||
@ApiOperation({ summary: 'Cancel a running pipeline run' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async cancelPipelineRun(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('runId') runId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const normalizedRunId = this.normalizePipelineId(runId);
|
||||
|
||||
const status = await this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run`,
|
||||
user,
|
||||
);
|
||||
|
||||
if (status.length === 1) {
|
||||
throw new BadRequestException('The first pipeline cannot be canceled');
|
||||
}
|
||||
|
||||
return this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
|
||||
@ApiOperation({
|
||||
summary: 'Get pipeline run jobs',
|
||||
description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.',
|
||||
})
|
||||
@ApiOkResponse({
|
||||
description: 'DB-backed job runs for the selected pipeline run.',
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
jobs: {
|
||||
type: 'array',
|
||||
items: { type: 'object' },
|
||||
},
|
||||
},
|
||||
required: ['jobs'],
|
||||
},
|
||||
})
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineRunJobs(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('runId') runId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const decodedRunId = this.decodePathParam(runId);
|
||||
|
||||
return this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== JOBS - COLUMN EDITING ROUTES ====================
|
||||
|
||||
@Put('jobs/:jobId/input')
|
||||
@@ -969,53 +918,40 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Delete('pipelines/:pipelineId/inputs/:inputId')
|
||||
@ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||
@UseGuards(PipelineExecutionGuard)
|
||||
async deleteTable(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('inputId') inputId: string,
|
||||
@Body() body: { table_name: string },
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const tableName = body.table_name;
|
||||
const info = {
|
||||
customer_id: user.customer_id,
|
||||
customer: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
};
|
||||
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
||||
|
||||
this.logger.info('deleteTable: marking table as deleted', { inputId, tableName });
|
||||
const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||
this.logger.info('deleteTable: table marked as deleted', { inputId, tableName });
|
||||
|
||||
try {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId });
|
||||
const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
||||
this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length });
|
||||
|
||||
const job = platformPipeline?.jobs?.find((j: any) => j.input?.table_name === tableName);
|
||||
if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`);
|
||||
|
||||
this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id });
|
||||
await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user);
|
||||
this.logger.info('deleteTable: job deleted', { jobId: job.job_id });
|
||||
|
||||
return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at };
|
||||
} catch (error) {
|
||||
this.logger.error('deleteTable: platform-api delete failed, attempting rollback', { tableName, error: error.message });
|
||||
try {
|
||||
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('deleteTable: rollback failed', { tableName, error: rollbackError.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@Get('jobs/jdbc/:jobId')
|
||||
@ApiOperation({ summary: 'Get JDBC job details' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getJdbcJob(@Param('jobId') jobId: string, @User() user: RequestUser) {
|
||||
// Normalize job ID for Platform API (replace - with _)
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
return this.platformApiService.proxy('GET', `/jobs/jdbc/${normalizedJobId}`, user);
|
||||
}
|
||||
|
||||
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
||||
@Post('jobs/jdbc/:jobId/sync-mode')
|
||||
@ApiOperation({ summary: 'Update JDBC job sync mode' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updateJdbcSyncMode(
|
||||
@Param('jobId') jobId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
// Normalize job ID for Platform API (replace - with _)
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
|
||||
const result = await this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/jobs/jdbc/${normalizedJobId}/sync-mode`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
|
||||
// Sync to DynamoDB (pass raw jobId for pipeline extraction)
|
||||
await this.syncJdbcSyncModeToDynamoDB(jobId, body, user);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@Get('jobs/jdbc/configs/allowed_datatypes')
|
||||
@ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
|
||||
@@ -1028,6 +964,52 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== JOBS - SINGER REPLICATION ROUTES ====================
|
||||
|
||||
@Get('jobs/singer/:jobId')
|
||||
@ApiOperation({ summary: 'Get Singer job details' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getSingerJob(@Param('jobId') jobId: string, @User() user: RequestUser) {
|
||||
// Normalize job ID for Platform API (replace - with _)
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
return this.platformApiService.proxy('GET', `/jobs/singer/${normalizedJobId}`, user);
|
||||
}
|
||||
|
||||
@Post('jobs/singer/:jobId/sync-mode')
|
||||
@ApiOperation({ summary: 'Update Singer job sync mode' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updateSingerSyncMode(
|
||||
@Param('jobId') jobId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
// Normalize job ID for Platform API (replace - with _)
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
|
||||
const result = await this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/jobs/singer/${normalizedJobId}/sync-mode`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
|
||||
// Sync to DynamoDB (pass raw jobId for pipeline extraction)
|
||||
await this.syncSingerSyncModeToDynamoDB(jobId, body, user);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ==================== JOBS - S3 ROUTES ====================
|
||||
|
||||
@Get('jobs/s3/:jobId')
|
||||
@ApiOperation({ summary: 'Get S3 job details' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getS3Job(@Param('jobId') jobId: string, @User() user: RequestUser) {
|
||||
// Normalize job ID for Platform API (replace - with _)
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
return this.platformApiService.proxy('GET', `/jobs/s3/${normalizedJobId}`, user);
|
||||
}
|
||||
|
||||
// ==================== HEALTH ROUTE ====================
|
||||
|
||||
@Get('health')
|
||||
|
||||
@@ -7,11 +7,9 @@ import { PlatformApiService } from './platform-api.service';
|
||||
import { ElasticsearchModule } from '../../services/elasticsearch';
|
||||
import { DynamoDBModule } from '../../services/dynamodb';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
import { CatalogModule } from '../catalog/catalog.module';
|
||||
import { InputsModule } from '../inputs/inputs.module';
|
||||
|
||||
@Module({
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, InputsModule],
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
|
||||
controllers: [PlatformApiController],
|
||||
providers: [PlatformApiService, DadosferaLogger],
|
||||
exports: [PlatformApiService],
|
||||
|
||||
@@ -89,12 +89,6 @@ export class PlatformApiService {
|
||||
|
||||
// Propagate non-2xx responses as HttpExceptions
|
||||
if (response.status >= 400) {
|
||||
this.logger.error('Platform API upstream error' + JSON.stringify({
|
||||
status: response.status,
|
||||
data: response.data,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
}));
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
@@ -107,8 +101,6 @@ export class PlatformApiService {
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
this.logger.error(error)
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
export type ReleaseNoteDTO = {
|
||||
id: string;
|
||||
date: string;
|
||||
tag: string;
|
||||
title: string;
|
||||
visible: boolean;
|
||||
expiryDate: string;
|
||||
content: string;
|
||||
showEmojis: boolean;
|
||||
image?: string;
|
||||
link?: string;
|
||||
linkText?: string;
|
||||
};
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { ReleaseNoteController } from './release_note.controller';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
|
||||
describe('ReleaseNoteController', () => {
|
||||
let controller: ReleaseNoteController;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
controllers: [ReleaseNoteController],
|
||||
providers: [ReleaseNoteService],
|
||||
}).compile();
|
||||
|
||||
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(controller).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -1,26 +0,0 @@
|
||||
import { Controller, Get, Inject } from '@nestjs/common';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Controller('release_note')
|
||||
@Authenticated()
|
||||
export class ReleaseNoteController {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private readonly releaseNoteService: ReleaseNoteService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Get()
|
||||
async getLatestReleaseNote(@Language() language: LanguageEnum) {
|
||||
this.logger.info(`Fetching latest release note for language: ${language}`);
|
||||
return await this.releaseNoteService.getLatestReleaseNote(language);
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
import { ReleaseNoteController } from './release_note.controller';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Module({
|
||||
controllers: [ReleaseNoteController],
|
||||
providers: [ReleaseNoteService, DadosferaLogger]
|
||||
})
|
||||
export class ReleaseNoteModule {}
|
||||
@@ -1,18 +0,0 @@
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { ReleaseNoteService } from './release_note.service';
|
||||
|
||||
describe('ReleaseNoteService', () => {
|
||||
let service: ReleaseNoteService;
|
||||
|
||||
beforeEach(async () => {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
providers: [ReleaseNoteService],
|
||||
}).compile();
|
||||
|
||||
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
||||
});
|
||||
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -1,46 +0,0 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import axios, { AxiosInstance } from 'axios';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { ReleaseNoteDTO } from './dto/release_note.dto';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Injectable()
|
||||
export class ReleaseNoteService {
|
||||
client: AxiosInstance;
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
this.client = axios.create({
|
||||
baseURL: process.env.FIREBASE_BASE_URL,
|
||||
});
|
||||
}
|
||||
|
||||
async getLatestReleaseNote(lang: LanguageEnum) {
|
||||
try {
|
||||
const lng = lang.split('-');
|
||||
const language = lng[0] + '-' + lng[1].toUpperCase();
|
||||
|
||||
const endpoint = `/release_note/${language}.json`;
|
||||
const {
|
||||
data,
|
||||
status,
|
||||
config
|
||||
} = await this.client.get<ReleaseNoteDTO>(endpoint)
|
||||
this.logger.info(`Fetched release note for language: ${lang} with status: ${status}`);
|
||||
this.logger.info(`Request URL: ${config.baseURL}/${config.url}`);
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
this.logger.error(`Error fetching release note: ${error.message}`);
|
||||
|
||||
if (axios.isAxiosError(error)) {
|
||||
this.logger.error(`Axios error details: ${error.toJSON()}`);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+2
-5
@@ -1,8 +1,5 @@
|
||||
export interface Info {
|
||||
user_id: string;
|
||||
customer_id: string;
|
||||
customer: string;
|
||||
}
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export interface ICreateTransformationsRequest {
|
||||
transformations: Transformation[];
|
||||
info: Info;
|
||||
|
||||
@@ -125,13 +125,9 @@ export class DynamoDBService {
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const { Item } = await this.documentClient.send(getCommand);
|
||||
return Item as InputDocument | null;
|
||||
} catch (error) {
|
||||
this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
const { Item } = await this.documentClient.send(getCommand);
|
||||
|
||||
return Item as InputDocument | null;
|
||||
}
|
||||
|
||||
async deleteInput(clientId: string, inputId: string): Promise<void> {
|
||||
@@ -207,6 +203,7 @@ export class DynamoDBService {
|
||||
updatedTable.reference_column = changes.reference_column;
|
||||
}
|
||||
}
|
||||
|
||||
tables[tableIndex] = updatedTable;
|
||||
|
||||
// Save updated document
|
||||
@@ -234,5 +231,4 @@ export class DynamoDBService {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -358,81 +358,6 @@ export class ElasticsearchService {
|
||||
}
|
||||
}
|
||||
|
||||
private getDataAssetIndex(customerName: string): string {
|
||||
return `${customerName}_data_assets_catalog`;
|
||||
}
|
||||
|
||||
async findDataAssetByTable(
|
||||
customerName: string,
|
||||
tableName: string,
|
||||
tableSchema: string,
|
||||
): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> {
|
||||
const index = this.getDataAssetIndex(customerName);
|
||||
|
||||
this.logger.info('Elasticsearch: Searching data asset', {
|
||||
index,
|
||||
tableName,
|
||||
tableSchema,
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.post(`/${index}/_search`, {
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ term: { 'table_name.keyword': tableName.toUpperCase() } },
|
||||
{ term: { 'table_schema.keyword': tableSchema.toUpperCase() } },
|
||||
],
|
||||
},
|
||||
},
|
||||
size: 1,
|
||||
});
|
||||
|
||||
const hits = response.data.hits?.hits || [];
|
||||
if (hits.length === 0) {
|
||||
this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index });
|
||||
return null;
|
||||
}
|
||||
|
||||
return { ...hits[0]._source, _es_id: hits[0]._id };
|
||||
} catch (error) {
|
||||
this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async updateDataAsset(
|
||||
customerName: string,
|
||||
assetId: string,
|
||||
updates: Record<string, any>,
|
||||
): Promise<any> {
|
||||
const index = this.getDataAssetIndex(customerName);
|
||||
|
||||
this.logger.info('Elasticsearch: Updating data asset', {
|
||||
index,
|
||||
assetId,
|
||||
fields: Object.keys(updates),
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.post(
|
||||
`/${index}/_update/${assetId}`,
|
||||
{ doc: updates },
|
||||
{ params: { refresh: 'wait_for' } },
|
||||
);
|
||||
|
||||
this.logger.info('Elasticsearch: Data asset updated', {
|
||||
assetId,
|
||||
result: response.data.result,
|
||||
});
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.handleError('updateDataAsset', error, { assetId, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private handleError(
|
||||
operation: string,
|
||||
error: any,
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { NimbusService } from "./nimbus.service";
|
||||
import DadosferaLogger from "@dadosfera/dadosfera-logs";
|
||||
|
||||
@Module({
|
||||
providers: [NimbusService, DadosferaLogger],
|
||||
exports: [NimbusService],
|
||||
})
|
||||
export class NimbusServicesModule {}
|
||||
@@ -1,56 +0,0 @@
|
||||
import DadosferaLogger from "@dadosfera/dadosfera-logs";
|
||||
import { Inject, Injectable } from "@nestjs/common";
|
||||
import axios from "axios";
|
||||
|
||||
type TableUpdate = {
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class NimbusService {
|
||||
private logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
private buildUrl(customerName: string) {
|
||||
if (process.env.ENV === 'prd') {
|
||||
return `https://nimbus-${customerName}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
return `https://nimbus-${customerName}.${process.env.ENV.replace(
|
||||
'local',
|
||||
'stg',
|
||||
)}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
async renameTable(customerName: string, database: string, old: TableUpdate, update: TableUpdate) {
|
||||
const nimbusUrl = this.buildUrl(customerName);
|
||||
|
||||
const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`;
|
||||
|
||||
try {
|
||||
this.logger.info("Request for PATCH " + nimbusUrl + path);
|
||||
this.logger.info("Payload: " + JSON.stringify(update));
|
||||
const { data } = await axios.patch(nimbusUrl + path, {
|
||||
table_name: update.table_name,
|
||||
table_schema: update.table_schema
|
||||
})
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
return {
|
||||
message: error.message,
|
||||
database,
|
||||
old,
|
||||
update
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ interface IMetadata {
|
||||
details?: string;
|
||||
sensitive?: string;
|
||||
roles?: string[];
|
||||
customer_modules?: string[];
|
||||
is_data_manager?: boolean;
|
||||
access_token?: string;
|
||||
host?: string;
|
||||
|
||||
Reference in New Issue
Block a user