mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-02 12:44:48 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
095306d2c7 |
@@ -4,7 +4,6 @@
|
||||
|
||||
# Maestro
|
||||
|
||||
|
||||
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
|
||||
|
||||
```mermaid
|
||||
|
||||
@@ -113,6 +113,12 @@ spec:
|
||||
value: {{ .Values.maestro.platform_api_url }}
|
||||
- name: STORAGE_EXPLORER_API_URL
|
||||
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
|
||||
- name: AUTODRIVE_EXTRACTOR_API_URL
|
||||
value: {{ .Values.maestro.autodrive_extractor_api_url | quote }}
|
||||
- name: AUTODRIVE_CORE_API_URL
|
||||
value: {{ .Values.maestro.autodrive_core_api_url | quote }}
|
||||
- name: AUTODRIVE_ASSISTANT_API_URL
|
||||
value: {{ .Values.maestro.autodrive_assistant_api_url | quote }}
|
||||
- name: JWT_PRIVATE_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -48,6 +48,9 @@ maestro:
|
||||
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
||||
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
|
||||
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
|
||||
autodrive_extractor_api_url: "https://autodrive-extractor-api-{customer}.dadosfera.ai"
|
||||
autodrive_core_api_url: "https://autodrive-api-{customer}.dadosfera.ai"
|
||||
autodrive_assistant_api_url: "https://autodrive-assistant-api-{customer}.dadosfera.ai"
|
||||
dedicated_proxy: ""
|
||||
restricted_ip: ""
|
||||
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
|
||||
|
||||
+21
-106
@@ -4747,42 +4747,6 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/jobs/{jobId}/rename-tables": {
|
||||
"post": {
|
||||
"operationId": "PlatformApiController_renameJobTables",
|
||||
"summary": "Rename job output tables and sync to catalog",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "jobId",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Platform API"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/jobs/jdbc/configs/allowed_datatypes": {
|
||||
"get": {
|
||||
"operationId": "PlatformApiController_getJdbcAllowedDatatypes",
|
||||
@@ -10161,21 +10125,6 @@
|
||||
"entities"
|
||||
]
|
||||
},
|
||||
"Column": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"type"
|
||||
]
|
||||
},
|
||||
"TableColumns": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -10191,28 +10140,27 @@
|
||||
"references": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/Column"
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"destination": {
|
||||
"type": "object"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
},
|
||||
"identifier_columns": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"reference_column": {
|
||||
"$ref": "#/components/schemas/Column"
|
||||
"destination": {
|
||||
"type": "object"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"columns",
|
||||
"references",
|
||||
"identifier_columns",
|
||||
"destination",
|
||||
"type"
|
||||
]
|
||||
@@ -10518,7 +10466,7 @@
|
||||
"enabled"
|
||||
]
|
||||
},
|
||||
"CustomerLinkItem": {
|
||||
"CustomerLink": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"href": {
|
||||
@@ -10540,61 +10488,28 @@
|
||||
"description"
|
||||
]
|
||||
},
|
||||
"CustomerSidebarSection": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "object"
|
||||
},
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/CustomerSidebarMenuItem"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/CustomerSidebarLinkItem"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"title",
|
||||
"items"
|
||||
]
|
||||
},
|
||||
"CustomerLinksConfig": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"home": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/CustomerLinkItem"
|
||||
}
|
||||
},
|
||||
"sidebar": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/CustomerSidebarSection"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"CustomerLinksResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"links": {
|
||||
"$ref": "#/components/schemas/CustomerLinksConfig"
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/CustomerLink"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"links"
|
||||
]
|
||||
},
|
||||
"CustomerLinkRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"links": {
|
||||
"$ref": "#/components/schemas/CustomerLinksConfig"
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/CustomerLink"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
Generated
+7
-79
@@ -17,7 +17,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
|
||||
"@dadosfera/protospack-v2": "3.38.0-beta.28",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -47,7 +47,6 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
@@ -1745,9 +1744,10 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.38.0-beta.30",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz",
|
||||
"integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==",
|
||||
"version": "3.38.0-beta.28",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
|
||||
"integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"rxjs": "^7.5.5"
|
||||
@@ -2928,20 +2928,6 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
|
||||
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
|
||||
"engines": [
|
||||
"node >= 0.8"
|
||||
],
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^2.2.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
|
||||
"version": "0.5.4",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
|
||||
@@ -3111,24 +3097,6 @@
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/multer": {
|
||||
"version": "1.4.4-lts.1",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
|
||||
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
|
||||
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.0.0",
|
||||
"concat-stream": "^1.5.2",
|
||||
"mkdirp": "^0.5.4",
|
||||
"object-assign": "^4.1.1",
|
||||
"type-is": "^1.6.4",
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
|
||||
@@ -3153,25 +3121,6 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
|
||||
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
|
||||
"dependencies": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
@@ -3246,19 +3195,6 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"dependencies": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/tslib": {
|
||||
"version": "2.5.3",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
|
||||
@@ -6573,6 +6509,7 @@
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
@@ -9191,11 +9128,6 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
|
||||
},
|
||||
"node_modules/isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
@@ -10793,6 +10725,7 @@
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
|
||||
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.6.0",
|
||||
@@ -11795,11 +11728,6 @@
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"node_modules/process-warning": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
|
||||
|
||||
+1
-2
@@ -35,7 +35,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
|
||||
"@dadosfera/protospack-v2": "3.38.0-beta.28",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -65,7 +65,6 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
|
||||
@@ -35,6 +35,9 @@ import { ShareMetadataModule } from './modules/share-metadata/share-metadata.mod
|
||||
import { ApiKeyModule } from './modules/api-key/api-key.module';
|
||||
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
|
||||
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
|
||||
import { AutodriveExtractorModule } from './modules/autodrive-extractor/autodrive-extractor.module';
|
||||
import { AutodriveCoreModule } from './modules/autodrive-core/autodrive-core.module';
|
||||
import { AutodriveAssistantModule } from './modules/autodrive-assistant/autodrive-assistant.module';
|
||||
|
||||
@Module({
|
||||
providers: [
|
||||
@@ -77,6 +80,9 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor
|
||||
NetworkPolicyModule,
|
||||
PlatformApiModule,
|
||||
StorageExplorerModule,
|
||||
AutodriveExtractorModule,
|
||||
AutodriveCoreModule,
|
||||
AutodriveAssistantModule,
|
||||
//Always leave HealthModule last, so it is on the bottom of swagger
|
||||
HealthModule,
|
||||
],
|
||||
|
||||
@@ -697,6 +697,93 @@ export const PERMISSIONS_GROUPS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_EXTRACTOR: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Extractor',
|
||||
'en-us': 'Autodrive Extractor',
|
||||
'es-es': 'Autodrive Extractor',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 53,
|
||||
claim: 'autodrive-extractor:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Extractor',
|
||||
'en-us': 'Read Autodrive Extractor data',
|
||||
'es-es': 'Leer datos del Autodrive Extractor',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 54,
|
||||
claim: 'autodrive-extractor:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Extractor',
|
||||
'en-us': 'Write Autodrive Extractor data',
|
||||
'es-es': 'Escribir datos en Autodrive Extractor',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_CORE: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Core',
|
||||
'en-us': 'Autodrive Core',
|
||||
'es-es': 'Autodrive Core',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 55,
|
||||
claim: 'autodrive-core:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Core',
|
||||
'en-us': 'Read Autodrive Core data',
|
||||
'es-es': 'Leer datos del Autodrive Core',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 56,
|
||||
claim: 'autodrive-core:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Core',
|
||||
'en-us': 'Write Autodrive Core data',
|
||||
'es-es': 'Escribir datos en Autodrive Core',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_ASSISTANT: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Assistant',
|
||||
'en-us': 'Autodrive Assistant',
|
||||
'es-es': 'Autodrive Assistant',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 57,
|
||||
claim: 'autodrive-assistant:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Assistant',
|
||||
'en-us': 'Read Autodrive Assistant data',
|
||||
'es-es': 'Leer datos del Autodrive Assistant',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 58,
|
||||
claim: 'autodrive-assistant:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Assistant',
|
||||
'en-us': 'Write Autodrive Assistant data',
|
||||
'es-es': 'Escribir datos en Autodrive Assistant',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
export interface DadosferaModule {
|
||||
name: string;
|
||||
|
||||
@@ -111,4 +111,3 @@ function configureSwagger(app: INestApplication) {
|
||||
);
|
||||
}
|
||||
bootstrap();
|
||||
|
||||
|
||||
@@ -478,7 +478,6 @@ export class AuthController {
|
||||
@Get('me')
|
||||
async getMe(@Req() req: Request, @Res() res: Response) {
|
||||
this.logger.info('GET /auth/me ')
|
||||
this.logger.info(JSON.stringify(req.headers));
|
||||
|
||||
// Check for API key header first
|
||||
const apiKey = req.get('X-Api-key');
|
||||
@@ -503,7 +502,7 @@ export class AuthController {
|
||||
const accessToken = req.cookies['ddf-auth'];
|
||||
const refreshToken = req.cookies['ddf-refresh-auth'];
|
||||
const userId = req.cookies['ddf-user-id'];
|
||||
const resourceHost = req.headers["x-original-url"] as string || "" ;
|
||||
const resourceHost = req.headers["host"]
|
||||
|
||||
const hasUserSession = Boolean(accessToken) && Boolean(userId);
|
||||
this.logger.info('Has User Session: ' + hasUserSession);
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_ASSISTANT_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_ASSISTANT_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_ASSISTANT_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_ASSISTANT_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,201 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveAssistantService } from './autodrive-assistant.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Assistant')
|
||||
@Controller('autodrive-assistant')
|
||||
@Authenticated()
|
||||
export class AutodriveAssistantController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveAssistantService: AutodriveAssistantService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// ASSISTANTS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List assistants' })
|
||||
@Get('assistants')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async listAssistants(
|
||||
@Query('limit') limit: number,
|
||||
@Query('offset') offset: number,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
'/assistants',
|
||||
user,
|
||||
undefined,
|
||||
{ limit, offset },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create assistant' })
|
||||
@Post('assistants')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async createAssistant(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'POST',
|
||||
'/assistants',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get assistant by ID' })
|
||||
@Get('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async getAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update assistant' })
|
||||
@Put('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async updateAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'PUT',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete assistant' })
|
||||
@Delete('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async deleteAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'DELETE',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// KNOWLEDGE BASE
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List knowledge bases linked to assistant' })
|
||||
@Get('assistants/:assistantId/knowledge-bases')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async listKnowledgeBases(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/assistants/${assistantId}/knowledge-bases`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update knowledge base associations for assistant' })
|
||||
@Put('assistants/:assistantId/knowledge-bases')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async updateKnowledgeBases(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'PUT',
|
||||
`/assistants/${assistantId}/knowledge-bases`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// QUESTIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Ask question via assistant' })
|
||||
@Post('dataset/:datasetId/ai_question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async aiQuestion(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/ai_question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get AI question answer' })
|
||||
@Get('dataset/:datasetId/ai_question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async getAiQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/ai_question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UTILITY
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveAssistantService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveAssistantService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveAssistantController } from './autodrive-assistant.controller';
|
||||
import { AutodriveAssistantService } from './autodrive-assistant.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveAssistantController],
|
||||
providers: [AutodriveAssistantService, DadosferaLogger],
|
||||
exports: [AutodriveAssistantService],
|
||||
})
|
||||
export class AutodriveAssistantModule {}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_ASSISTANT_CONFIG } from './autodrive-assistant.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveAssistantService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive assistant operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_ASSISTANT_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-assistant', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_ASSISTANT_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Assistant API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Assistant API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Assistant API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_CORE_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_CORE_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_CORE_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_CORE_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,286 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
UseInterceptors,
|
||||
UploadedFiles,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
|
||||
import { FilesInterceptor } from '@nestjs/platform-express';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import FormData from 'form-data';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveCoreService } from './autodrive-core.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Core')
|
||||
@Controller('autodrive-core')
|
||||
@Authenticated()
|
||||
export class AutodriveCoreController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveCoreService: AutodriveCoreService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DATASET MANAGEMENT
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List datasets' })
|
||||
@Get('datasets')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async listDatasets(
|
||||
@Query('name') name: string,
|
||||
@Query('limit') limit: number,
|
||||
@Query('offset') offset: number,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
'/datasets',
|
||||
user,
|
||||
undefined,
|
||||
{ name, limit, offset },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get dataset by ID' })
|
||||
@Get('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Add documents to existing dataset' })
|
||||
@Put('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async updateDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'PUT',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete dataset' })
|
||||
@Delete('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async deleteDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'DELETE',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Append documents from URLs to dataset' })
|
||||
@Post('dataset/:datasetId/append-from-url')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async appendFromUrl(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/append-from-url`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get dataset creation/update logs' })
|
||||
@Get('dataset/:datasetId/logs')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getDatasetLogs(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/logs`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// FILE OPERATIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Create vector dataset from uploaded files' })
|
||||
@Post('upload')
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@UseInterceptors(FilesInterceptor('files'))
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async upload(
|
||||
@UploadedFiles() files: Array<Express.Multer.File>,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const formData = new FormData();
|
||||
if (files && files.length > 0) {
|
||||
files.forEach((file) => {
|
||||
formData.append('files', file.buffer, {
|
||||
filename: file.originalname,
|
||||
contentType: file.mimetype,
|
||||
});
|
||||
});
|
||||
}
|
||||
// Forward additional body fields
|
||||
if (body) {
|
||||
Object.entries(body).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null && key !== 'files') {
|
||||
formData.append(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
return this.autodriveCoreService.proxyFormData(
|
||||
'POST',
|
||||
'/upload',
|
||||
user,
|
||||
formData,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create vector dataset from URLs' })
|
||||
@Post('upload-from-url')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async uploadFromUrl(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
'/upload-from-url',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// QUERY OPERATIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Search dataset using semantic search' })
|
||||
@Post('dataset/:datasetId/question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async question(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get semantic search results' })
|
||||
@Get('dataset/:datasetId/question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Ask AI question on dataset' })
|
||||
@Post('dataset/:datasetId/ai_question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async aiQuestion(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/ai_question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get AI question answer' })
|
||||
@Get('dataset/:datasetId/ai_question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getAiQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/ai_question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// SYSTEM
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get frontend configuration' })
|
||||
@Get('config')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getConfig(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/config', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get usage metrics' })
|
||||
@Get('usage')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getUsage(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/usage', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveCoreController } from './autodrive-core.controller';
|
||||
import { AutodriveCoreService } from './autodrive-core.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveCoreController],
|
||||
providers: [AutodriveCoreService, DadosferaLogger],
|
||||
exports: [AutodriveCoreService],
|
||||
})
|
||||
export class AutodriveCoreModule {}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_CORE_CONFIG } from './autodrive-core.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveCoreService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive core operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_CORE_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-core', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_CORE_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Core API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Core API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Core API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
|
||||
async proxyFormData(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
formData: any,
|
||||
query?: Record<string, any>,
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive core operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_CORE_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
...formData.getHeaders?.(),
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying form data request to autodrive-core', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: formData,
|
||||
timeout: AUTODRIVE_CORE_CONFIG.timeout,
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Core API form data proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_EXTRACTOR_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_EXTRACTOR_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_EXTRACTOR_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_EXTRACTOR_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,491 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
UseInterceptors,
|
||||
UploadedFiles,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
|
||||
import { FilesInterceptor } from '@nestjs/platform-express';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import FormData from 'form-data';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveExtractorService } from './autodrive-extractor.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Extractor')
|
||||
@Controller('autodrive-extractor')
|
||||
@Authenticated()
|
||||
export class AutodriveExtractorController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveExtractorService: AutodriveExtractorService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LEGACY - NPL Extraction routes (to be removed)
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Extract features from dataset using NPL' })
|
||||
@Post('npl/datasets/:datasetId/extract')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplExtract(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
`/npl/datasets/${datasetId}/extract`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] List NPL extractions for a dataset' })
|
||||
@Get('npl/datasets/:datasetId/extractions')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplListExtractions(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Query('status') status: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions`,
|
||||
user,
|
||||
undefined,
|
||||
{ status },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL extraction details' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetExtraction(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL extraction to CSV' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportExtractionCsv(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL extraction to XLSX' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/xlsx')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportExtractionXlsx(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/xlsx`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Batch export all NPL extractions to CSV' })
|
||||
@Get('npl/datasets/:datasetId/extractions/export/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplBatchExportCsv(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/export/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Batch export all NPL extractions to XLSX' })
|
||||
@Get('npl/datasets/:datasetId/extractions/export/xlsx')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplBatchExportXlsx(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/export/xlsx`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL extraction report' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/report')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetExtractionReport(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/report`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LEGACY - NPL Configuration routes (to be removed)
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration' })
|
||||
@Get('npl/config')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration summary' })
|
||||
@Get('npl/config/summary')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigSummary(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/summary', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] List NPL configuration fields' })
|
||||
@Get('npl/config/fields')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigFields(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/fields', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration field by ID' })
|
||||
@Get('npl/config/fields/:fieldId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigField(
|
||||
@Param('fieldId') fieldId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/config/fields/${fieldId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Update NPL configuration field' })
|
||||
@Put('npl/config/fields/:fieldId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplUpdateConfigField(
|
||||
@Param('fieldId') fieldId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
`/npl/config/fields/${fieldId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL RAG queries' })
|
||||
@Get('npl/config/rag-queries')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetRagQueries(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/rag-queries', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL system prompt' })
|
||||
@Get('npl/config/system-prompt')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetSystemPrompt(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/system-prompt', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Update NPL system prompt' })
|
||||
@Put('npl/config/system-prompt')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplUpdateSystemPrompt(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
'/npl/config/system-prompt',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL source priorities' })
|
||||
@Get('npl/config/source-priorities')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetSourcePriorities(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/source-priorities', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Reset NPL configuration to defaults' })
|
||||
@Post('npl/config/reset')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplResetConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('POST', '/npl/config/reset', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL configuration' })
|
||||
@Get('npl/config/export')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/export', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Import NPL configuration' })
|
||||
@Post('npl/config/import')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplImportConfig(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/npl/config/import',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// EXTRACTION TEMPLATES
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Seed NPL Brasil extraction template' })
|
||||
@Post('templates/seed-npl')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async seedNplTemplate(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/templates/seed-npl',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'List extraction templates' })
|
||||
@Get('templates')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async listTemplates(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/templates', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create extraction template' })
|
||||
@Post('templates')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async createTemplate(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/templates',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction template by ID' })
|
||||
@Get('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update extraction template' })
|
||||
@Put('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async updateTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete extraction template' })
|
||||
@Delete('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async deleteTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'DELETE',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Import extraction templates from file' })
|
||||
@Post('templates/import')
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@UseInterceptors(FilesInterceptor('file'))
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async importTemplates(
|
||||
@UploadedFiles() files: Array<Express.Multer.File>,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const formData = new FormData();
|
||||
if (files && files.length > 0) {
|
||||
files.forEach((file) => {
|
||||
formData.append('file', file.buffer, {
|
||||
filename: file.originalname,
|
||||
contentType: file.mimetype,
|
||||
});
|
||||
});
|
||||
}
|
||||
return this.autodriveExtractorService.proxyFormData(
|
||||
'POST',
|
||||
'/templates/import',
|
||||
user,
|
||||
formData,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// GENERIC EXTRACTION
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Start template-based extraction on dataset' })
|
||||
@Post('extraction/datasets/:datasetId/extract')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async startExtraction(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
`/extraction/datasets/${datasetId}/extract`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'List extraction jobs for a dataset' })
|
||||
@Get('extraction/datasets/:datasetId/jobs')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async listExtractionJobs(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Query('status') status: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/datasets/${datasetId}/jobs`,
|
||||
user,
|
||||
undefined,
|
||||
{ status },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction job details' })
|
||||
@Get('extraction/jobs/:jobId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getExtractionJob(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction job report' })
|
||||
@Get('extraction/jobs/:jobId/report')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getExtractionJobReport(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}/report`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Export extraction job to CSV' })
|
||||
@Get('extraction/jobs/:jobId/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async exportExtractionJobCsv(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UTILITY
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveExtractorController } from './autodrive-extractor.controller';
|
||||
import { AutodriveExtractorService } from './autodrive-extractor.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveExtractorController],
|
||||
providers: [AutodriveExtractorService, DadosferaLogger],
|
||||
exports: [AutodriveExtractorService],
|
||||
})
|
||||
export class AutodriveExtractorModule {}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_EXTRACTOR_CONFIG } from './autodrive-extractor.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveExtractorService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive extractor operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_EXTRACTOR_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-extractor', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_EXTRACTOR_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Extractor API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Extractor API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Extractor API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
|
||||
async proxyFormData(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
formData: any,
|
||||
query?: Record<string, any>,
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive extractor operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_EXTRACTOR_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
...formData.getHeaders?.(),
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying form data request to autodrive-extractor', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: formData,
|
||||
timeout: AUTODRIVE_EXTRACTOR_CONFIG.timeout,
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Extractor API form data proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -734,64 +734,6 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async renameTableOnNimbus(
|
||||
nimbusUrl: string,
|
||||
nimbusId: number,
|
||||
changes: { table_name?: string; table_schema?: string; display_name?: string },
|
||||
): Promise<void> {
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
|
||||
this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
|
||||
await axios.patch(endpoint, changes);
|
||||
}
|
||||
|
||||
async renameColumnMetadataOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
|
||||
const { data: columns } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(columns) ? columns : [];
|
||||
|
||||
for (const column of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
|
||||
}
|
||||
|
||||
async renameDataPreviewOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
|
||||
const { data: previews } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(previews) ? previews : [];
|
||||
|
||||
for (const preview of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
|
||||
}
|
||||
|
||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||
const customer_name_raw = metadata.get('customer_name');
|
||||
|
||||
|
||||
@@ -9,12 +9,12 @@ import {
|
||||
} from '@nestjs/common';
|
||||
|
||||
import { firstValueFrom, lastValueFrom } from 'rxjs';
|
||||
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||
import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
import { CustomerLinksConfig } from './dtos/customers';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import {
|
||||
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
|
||||
)
|
||||
}
|
||||
|
||||
async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
|
||||
async getLinks(customerId: string) {
|
||||
try {
|
||||
const result = await lastValueFrom(
|
||||
this.customerService.CustomerGetLinks({ customerId }),
|
||||
this.customerService.CustomerFindOneById({ id: customerId }),
|
||||
);
|
||||
return (result.links as CustomerLinksConfig) || null;
|
||||
return result.customer?.links || [];
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
|
||||
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async setLinks(customerId: string, links: CustomerLinksConfig) {
|
||||
async setLinks(customerId: string, links: Link[]) {
|
||||
if (!customerId || !links) {
|
||||
throw new HttpException(null, HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
try {
|
||||
return await firstValueFrom(
|
||||
this.customerService.CustomerSetLinks({
|
||||
customerId,
|
||||
links: links as CustomerSetLinksRequest['links'],
|
||||
}),
|
||||
this.customerService.CustomerUpdate({
|
||||
id: customerId,
|
||||
links,
|
||||
} as CustomerUpdateRequest),
|
||||
);
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
|
||||
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class CustomerLinkItem {
|
||||
export class CustomerLink implements Link {
|
||||
@ApiProperty()
|
||||
href: string;
|
||||
@ApiProperty()
|
||||
@@ -8,59 +9,15 @@ export class CustomerLinkItem {
|
||||
@ApiProperty()
|
||||
description: string;
|
||||
@ApiPropertyOptional()
|
||||
iconSrc?: string;
|
||||
iconSrc: string;
|
||||
}
|
||||
|
||||
export class CustomerSidebarLinkItem {
|
||||
@ApiProperty()
|
||||
type: 'link';
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiProperty()
|
||||
link: string;
|
||||
@ApiPropertyOptional()
|
||||
icon?: string;
|
||||
}
|
||||
|
||||
export class CustomerSidebarMenuItem {
|
||||
@ApiProperty()
|
||||
type: 'menu';
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiPropertyOptional()
|
||||
icon?: string;
|
||||
@ApiProperty({ type: [CustomerSidebarLinkItem] })
|
||||
items: CustomerSidebarLinkItem[];
|
||||
}
|
||||
|
||||
export class CustomerSidebarSection {
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiProperty({
|
||||
type: 'array',
|
||||
items: {
|
||||
oneOf: [
|
||||
{ $ref: '#/components/schemas/CustomerSidebarMenuItem' },
|
||||
{ $ref: '#/components/schemas/CustomerSidebarLinkItem' },
|
||||
],
|
||||
},
|
||||
})
|
||||
items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
|
||||
}
|
||||
|
||||
export class CustomerLinksConfig {
|
||||
@ApiPropertyOptional({ type: [CustomerLinkItem] })
|
||||
home?: CustomerLinkItem[];
|
||||
@ApiPropertyOptional({ type: [CustomerSidebarSection] })
|
||||
sidebar?: CustomerSidebarSection[];
|
||||
}
|
||||
|
||||
export class CustomerLinkRequest {
|
||||
@ApiProperty({ type: CustomerLinksConfig })
|
||||
links: CustomerLinksConfig;
|
||||
@ApiProperty({ type: [CustomerLink] })
|
||||
links: CustomerLink[];
|
||||
}
|
||||
|
||||
export class CustomerLinksResponse {
|
||||
@ApiPropertyOptional({ type: CustomerLinksConfig })
|
||||
links?: CustomerLinksConfig;
|
||||
}
|
||||
@ApiProperty({ type: [CustomerLink] })
|
||||
links: CustomerLink[];
|
||||
}
|
||||
|
||||
|
||||
@@ -11,19 +11,17 @@ export class TableColumns {
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
columns: string[];
|
||||
@ApiPropertyOptional({ type: [Column] })
|
||||
@ApiProperty()
|
||||
references: Column[];
|
||||
@ApiProperty()
|
||||
identifier_columns: string[];
|
||||
@ApiProperty()
|
||||
destination: Record<'raw' | 'qualify', {
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
}> | null;
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
@ApiPropertyOptional({ type: [String] })
|
||||
identifier_columns?: string[];
|
||||
@ApiPropertyOptional({ type: Column })
|
||||
reference_column?: Column;
|
||||
}
|
||||
export class AvailableEntity {
|
||||
@ApiProperty()
|
||||
|
||||
@@ -164,11 +164,6 @@ export class InputsService {
|
||||
const inputCreateGenericRequest: InputCreateGenericRequest = {
|
||||
input: {
|
||||
...body,
|
||||
tables: (body.tables || []).map((table) => ({
|
||||
...table,
|
||||
identifier_columns: table.identifier_columns || [],
|
||||
reference_column: table.reference_column || table.references?.[0],
|
||||
})),
|
||||
},
|
||||
info,
|
||||
};
|
||||
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
Query,
|
||||
Inject,
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
@@ -26,8 +25,6 @@ import { ElasticsearchService } from '../../services/elasticsearch';
|
||||
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
|
||||
import { CustomersService } from '../customers/customers.service';
|
||||
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
||||
import { CatalogService } from '../catalog/catalog.service';
|
||||
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
||||
import { ValidationTableDTO } from './platform-api.dto';
|
||||
|
||||
|
||||
@@ -38,11 +35,6 @@ type ValidateTablesDTO = {
|
||||
}>
|
||||
}
|
||||
|
||||
type RenameTablesBody = {
|
||||
raw?: { table_name: string; table_schema: string };
|
||||
qualify?: { table_name: string; table_schema: string };
|
||||
}
|
||||
|
||||
@ApiTags('Platform API')
|
||||
@Controller('platform')
|
||||
export class PlatformApiController {
|
||||
@@ -53,7 +45,6 @@ export class PlatformApiController {
|
||||
private readonly elasticsearchService: ElasticsearchService,
|
||||
private readonly dynamoDBService: DynamoDBService,
|
||||
private readonly customersService: CustomersService,
|
||||
private readonly catalogService: CatalogService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
@@ -85,23 +76,6 @@ export class PlatformApiController {
|
||||
return jobId?.replace(/-/g, '_') || '';
|
||||
}
|
||||
|
||||
private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise<any> {
|
||||
const connectorTypes = ['jdbc', 'singer', 's3'];
|
||||
for (const type of connectorTypes) {
|
||||
try {
|
||||
const job = await this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/jobs/${type}/${normalizedJobId}`,
|
||||
user,
|
||||
);
|
||||
return job;
|
||||
} catch (error) {
|
||||
// Continue to next connector type
|
||||
}
|
||||
}
|
||||
throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the pipeline ID (base UUID) from a job ID.
|
||||
* Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1").
|
||||
@@ -979,192 +953,6 @@ export class PlatformApiController {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Post('jobs/:jobId/rename-tables')
|
||||
@ApiOperation({ summary: 'Rename job output tables and sync to catalog' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async renameJobTables(
|
||||
@Param('jobId') jobId: string,
|
||||
@Body() body: RenameTablesBody,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
|
||||
const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user);
|
||||
|
||||
const result = await this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/jobs/${normalizedJobId}/rename-tables`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.syncTableRenameToCatalog(jobId, body, currentJob, user);
|
||||
} catch (error) {
|
||||
this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message });
|
||||
|
||||
const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {});
|
||||
if (reverseBody) {
|
||||
try {
|
||||
await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody);
|
||||
this.logger.info('Snowflake rename rolled back', { jobId });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message });
|
||||
}
|
||||
}
|
||||
|
||||
throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private buildSnowflakeRollbackBody(
|
||||
body: RenameTablesBody,
|
||||
outputConfig: any,
|
||||
): RenameTablesBody | null {
|
||||
const reverse: RenameTablesBody = {};
|
||||
|
||||
if (body.raw) {
|
||||
const nested = outputConfig.raw;
|
||||
const oldTableName = nested?.table_name || outputConfig.table_name;
|
||||
const oldTableSchema = nested?.table_schema || 'PUBLIC';
|
||||
if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema };
|
||||
}
|
||||
|
||||
if (body.qualify) {
|
||||
const nested = outputConfig.qualify;
|
||||
if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' };
|
||||
}
|
||||
|
||||
return Object.keys(reverse).length > 0 ? reverse : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync table rename to Elasticsearch and Nimbus.
|
||||
*
|
||||
* For each target (raw, qualify):
|
||||
* 1. Resolve old table name from output_config
|
||||
* 2. Find the ES data asset by pipeline + table + schema
|
||||
* 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview
|
||||
* 4. If any step fails, rollback all completed steps for that target
|
||||
*/
|
||||
private async syncTableRenameToCatalog(
|
||||
jobId: string,
|
||||
body: RenameTablesBody,
|
||||
currentJob: any,
|
||||
user: RequestUser,
|
||||
): Promise<void> {
|
||||
const pipelineId = this.extractPipelineIdFromJobId(jobId);
|
||||
const outputConfig = currentJob.output_config || {};
|
||||
const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } });
|
||||
const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`;
|
||||
|
||||
const targets = this.buildRenameTargets(body, outputConfig);
|
||||
|
||||
for (const { key, oldTableName, oldTableSchema, newValues } of targets) {
|
||||
const rollbackSteps: Array<() => Promise<void>> = [];
|
||||
|
||||
try {
|
||||
const dataAsset = await this.elasticsearchService.findDataAssetByTable(
|
||||
user.customer_name, oldTableName, oldTableSchema,
|
||||
);
|
||||
|
||||
if (!dataAsset) {
|
||||
this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema });
|
||||
continue;
|
||||
}
|
||||
|
||||
const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset;
|
||||
const oldValues = { table_name: oldTableName, table_schema: oldTableSchema };
|
||||
|
||||
// ES update
|
||||
const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name };
|
||||
await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields);
|
||||
rollbackSteps.push(() => this.elasticsearchService.updateDataAsset(
|
||||
user.customer_name, esAssetId,
|
||||
{ name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName },
|
||||
));
|
||||
|
||||
const newTableNameUpper = newValues.table_name.toUpperCase();
|
||||
const newTableSchemaUpper = newValues.table_schema.toUpperCase();
|
||||
const oldTableNameUpper = oldTableName.toUpperCase();
|
||||
const oldTableSchemaUpper = oldTableSchema.toUpperCase();
|
||||
|
||||
// Nimbus table-metadata
|
||||
if (nimbusId) {
|
||||
await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper });
|
||||
rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper }));
|
||||
}
|
||||
|
||||
// Nimbus column-metadata
|
||||
await this.catalogService.renameColumnMetadataOnNimbus(
|
||||
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
|
||||
);
|
||||
rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus(
|
||||
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
|
||||
));
|
||||
|
||||
// Nimbus data-preview
|
||||
await this.catalogService.renameDataPreviewOnNimbus(
|
||||
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
|
||||
);
|
||||
rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus(
|
||||
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
|
||||
));
|
||||
|
||||
this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name });
|
||||
} catch (error) {
|
||||
this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message });
|
||||
await this.executeRollback(rollbackSteps, key, jobId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private buildRenameTargets(
|
||||
body: RenameTablesBody,
|
||||
outputConfig: any,
|
||||
): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> {
|
||||
const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' };
|
||||
const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = [];
|
||||
|
||||
for (const key of ['raw', 'qualify'] as const) {
|
||||
if (!body[key]) continue;
|
||||
|
||||
const nested = outputConfig[key];
|
||||
|
||||
// qualify: only sync if output_config.qualify already exists
|
||||
if (key === 'qualify' && !nested?.table_name) continue;
|
||||
|
||||
const oldTableName = nested?.table_name || outputConfig.table_name;
|
||||
if (!oldTableName) continue;
|
||||
|
||||
targets.push({
|
||||
key,
|
||||
oldTableName,
|
||||
oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key],
|
||||
newValues: body[key],
|
||||
});
|
||||
}
|
||||
|
||||
return targets;
|
||||
}
|
||||
|
||||
private async executeRollback(
|
||||
steps: Array<() => Promise<void>>,
|
||||
targetKey: string,
|
||||
jobId: string,
|
||||
): Promise<void> {
|
||||
for (const rollback of steps.reverse()) {
|
||||
try {
|
||||
await rollback();
|
||||
} catch (error) {
|
||||
this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Get('jobs/jdbc/configs/allowed_datatypes')
|
||||
@ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
|
||||
@@ -7,10 +7,9 @@ import { PlatformApiService } from './platform-api.service';
|
||||
import { ElasticsearchModule } from '../../services/elasticsearch';
|
||||
import { DynamoDBModule } from '../../services/dynamodb';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
import { CatalogModule } from '../catalog/catalog.module';
|
||||
|
||||
@Module({
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule],
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
|
||||
controllers: [PlatformApiController],
|
||||
providers: [PlatformApiService, DadosferaLogger],
|
||||
exports: [PlatformApiService],
|
||||
|
||||
@@ -89,12 +89,6 @@ export class PlatformApiService {
|
||||
|
||||
// Propagate non-2xx responses as HttpExceptions
|
||||
if (response.status >= 400) {
|
||||
this.logger.error('Platform API upstream error', {
|
||||
status: response.status,
|
||||
data: response.data,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
|
||||
@@ -358,81 +358,6 @@ export class ElasticsearchService {
|
||||
}
|
||||
}
|
||||
|
||||
private getDataAssetIndex(customerName: string): string {
|
||||
return `${customerName}_data_assets_catalog`;
|
||||
}
|
||||
|
||||
async findDataAssetByTable(
|
||||
customerName: string,
|
||||
tableName: string,
|
||||
tableSchema: string,
|
||||
): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> {
|
||||
const index = this.getDataAssetIndex(customerName);
|
||||
|
||||
this.logger.info('Elasticsearch: Searching data asset', {
|
||||
index,
|
||||
tableName,
|
||||
tableSchema,
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.post(`/${index}/_search`, {
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ term: { 'table_name.keyword': tableName.toUpperCase() } },
|
||||
{ term: { 'table_schema.keyword': tableSchema.toUpperCase() } },
|
||||
],
|
||||
},
|
||||
},
|
||||
size: 1,
|
||||
});
|
||||
|
||||
const hits = response.data.hits?.hits || [];
|
||||
if (hits.length === 0) {
|
||||
this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index });
|
||||
return null;
|
||||
}
|
||||
|
||||
return { ...hits[0]._source, _es_id: hits[0]._id };
|
||||
} catch (error) {
|
||||
this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async updateDataAsset(
|
||||
customerName: string,
|
||||
assetId: string,
|
||||
updates: Record<string, any>,
|
||||
): Promise<any> {
|
||||
const index = this.getDataAssetIndex(customerName);
|
||||
|
||||
this.logger.info('Elasticsearch: Updating data asset', {
|
||||
index,
|
||||
assetId,
|
||||
fields: Object.keys(updates),
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.post(
|
||||
`/${index}/_update/${assetId}`,
|
||||
{ doc: updates },
|
||||
{ params: { refresh: 'wait_for' } },
|
||||
);
|
||||
|
||||
this.logger.info('Elasticsearch: Data asset updated', {
|
||||
assetId,
|
||||
result: response.data.result,
|
||||
});
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.handleError('updateDataAsset', error, { assetId, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private handleError(
|
||||
operation: string,
|
||||
error: any,
|
||||
|
||||
Reference in New Issue
Block a user