Compare commits

...
Author SHA1 Message Date
viniciusgadeaandClaude Sonnet 4.6 b7171556b8 FIX: add multer@2.0.2 and update package-lock.json — missing transitive dep of protospack-v2@3.38.0-beta.30
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-19 13:58:02 -03:00
viniciusgadea 74b3bd6b46 FEAT: update CustomerSidebarSection and related DTOs to use object type for title; update protospack-v2 version; update from id to customerId 2026-03-19 08:14:56 -03:00
viniciusgadea bf19d29a1d FIX: docsfera.json 2026-03-17 15:38:35 -03:00
viniciusgadeaandClaude Sonnet 4.6 6523f707e3 FIX: remove duplicate identifier_columns declaration in TableColumns DTO
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-17 15:37:08 -03:00
viniciusgadeaandClaude Sonnet 4.6 8b0bf84d34 FIX: resolve merge conflict in package.json for protospack-v2 version
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-17 15:34:12 -03:00
viniciusgadea 9ef4c51ba1 FEAT: update CustomerSidebarSection to support both menu and link items 2026-03-17 15:25:55 -03:00
vinicius gadea 19521489fa Merge branch 'beta' into feat/sidebar-menu 2026-03-17 10:22:37 -03:00
Rafael Santana 2ce9aad005 Merge pull request #460 from dadosfera/force-deploy
FIX: uppercase table_name and table_schema in Nimbus rename calls
2026-03-16 10:42:41 -03:00
RafaelandClaude Opus 4.6 051fb6e4dd FIX: uppercase table_name and table_schema in Nimbus rename calls
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-16 10:39:55 -03:00
Rafael Santana 2125884c6c Merge pull request #459 from dadosfera/force-deploy
FIX: uppercase table_name and table_schema in ES lookup
2026-03-16 10:19:02 -03:00
RafaelandClaude Opus 4.6 e3099aa2b2 FIX: uppercase table_name and table_schema in ES lookup
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-16 10:17:56 -03:00
Rafael Santana f4c9226ef9 Merge pull request #458 from dadosfera/force-deploy
FIX: rename-tables proxy path and ES lookup
2026-03-13 18:29:53 -03:00
RafaelandClaude Opus 4.6 12c61d9b5d FIX: rename-tables proxy path and ES lookup
- Fix proxy path: /jobs/jdbc/:jobId/rename-tables → /jobs/:jobId/rename-tables
- Remove pipeline_id from ES data asset lookup, search by table_name + table_schema only

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-13 18:08:39 -03:00
viniciusgadea ff3999a6aa FEAT: update customer links handling and DTOs for improved structure sidebar. update protospack-v2 2026-03-13 17:35:56 -03:00
Rafael Santana 209470482a Merge pull request #456 from dadosfera/force-deploy
UPDATE: force deployment of maestro
2026-03-12 18:03:19 -03:00
Rafael 99c2a9ecf5 UPDATE: force deployment of maestro 2026-03-12 18:02:49 -03:00
Rafael Santana c0f75d241f Merge pull request #449 from dadosfera/feat/rename-tables-catalog-sync
Feat/rename tables catalog sync
2026-03-12 17:57:44 -03:00
Rafael Santana 1e0fb78dff Merge branch 'beta' into feat/rename-tables-catalog-sync 2026-03-12 17:57:37 -03:00
Marcos Rodrigues Silva c26194554c Merge pull request #455 from dadosfera/fix/header-validation
Fix/header validation
2026-03-11 11:58:23 -03:00
Marcos Rodrigues Silva b70d37423d Merge pull request #454 from dadosfera/fix/header-validation
FIX: types
2026-03-11 10:27:31 -03:00
marcos-silva-rodrigues 4ddd5edcfd FIX: types 2026-03-11 10:26:33 -03:00
Marcos Rodrigues Silva 3bcbba9581 Merge pull request #453 from dadosfera/fix/header-validation
Fix/header validation
2026-03-10 17:35:31 -03:00
marcos-silva-rodrigues 67d47a9642 FIX: correct header 2026-03-10 17:35:05 -03:00
Marcos Rodrigues Silva 6f9c967c96 Merge pull request #452 from dadosfera/feat/qualify
FIX: stringify headers
2026-03-10 17:17:48 -03:00
marcos-silva-rodrigues b8bdc5beea FIX: stringify headers 2026-03-10 16:00:39 -03:00
Marcos Rodrigues Silva 269f70b309 Merge pull request #451 from dadosfera/feat/qualify
FIX: ghost commit
2026-03-10 15:22:36 -03:00
marcos-silva-rodrigues 93f452ae05 FIX: ghost commit 2026-03-10 15:19:18 -03:00
Marcos Rodrigues Silva e39378229f Merge pull request #450 from dadosfera/feat/qualify
feat: list headers
2026-03-10 14:43:43 -03:00
marcos-silva-rodrigues 61a4f724ef feat: list headers 2026-03-10 14:43:11 -03:00
RafaelandClaude Opus 4.6 38a9e21f5f feat: add rename-tables endpoint with catalog sync and rollback
Add POST /platform/jobs/:jobId/rename-tables that renames Snowflake
tables via platform-api and syncs the rename to Elasticsearch and
Nimbus (table-metadata, column-metadata, data-preview). If catalog
sync fails, all completed catalog steps are rolled back in reverse
order and the Snowflake rename is reverted.

- Support any connector type (jdbc, singer, s3) via getJobByAnyConnectorType
- Resolve old table names from output_config (raw/qualify)
- Skip qualify sync when output_config.qualify has no table_name
- Add findDataAssetByPipelineAndTable and updateDataAsset to ElasticsearchService
- Add renameTableOnNimbus, renameColumnMetadataOnNimbus, renameDataPreviewOnNimbus to CatalogService
- Add upstream error logging to PlatformApiService

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 17:29:12 -03:00
Marcos Rodrigues Silva 1a2153d62f Merge pull request #448 from dadosfera/fix/proxy-urls
Fix/proxy urls
2026-02-20 16:03:05 -03:00
Marcos Rodrigues Silva d25bfd147c Merge pull request #447 from dadosfera/fix/proxy-urls
Fix/proxy urls
2026-02-20 14:32:50 -03:00
marcos-silva-rodrigues d4ba45fd03 FIX: storage url 2026-02-20 14:28:55 -03:00
marcos-silva-rodrigues 82f1035a5e FIX: update storage port 2026-02-20 14:28:36 -03:00
15 changed files with 613 additions and 50 deletions
+1
View File
@@ -4,6 +4,7 @@
# Maestro
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
```mermaid
+103 -18
View File
@@ -4747,6 +4747,42 @@
]
}
},
"/platform/jobs/{jobId}/rename-tables": {
"post": {
"operationId": "PlatformApiController_renameJobTables",
"summary": "Rename job output tables and sync to catalog",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/jdbc/configs/allowed_datatypes": {
"get": {
"operationId": "PlatformApiController_getJdbcAllowedDatatypes",
@@ -10125,6 +10161,21 @@
"entities"
]
},
"Column": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string"
}
},
"required": [
"name",
"type"
]
},
"TableColumns": {
"type": "object",
"properties": {
@@ -10140,13 +10191,7 @@
"references": {
"type": "array",
"items": {
"type": "string"
}
},
"identifier_columns": {
"type": "array",
"items": {
"type": "string"
"$ref": "#/components/schemas/Column"
}
},
"destination": {
@@ -10154,13 +10199,20 @@
},
"type": {
"type": "string"
},
"identifier_columns": {
"type": "array",
"items": {
"type": "string"
}
},
"reference_column": {
"$ref": "#/components/schemas/Column"
}
},
"required": [
"name",
"columns",
"references",
"identifier_columns",
"destination",
"type"
]
@@ -10466,7 +10518,7 @@
"enabled"
]
},
"CustomerLink": {
"CustomerLinkItem": {
"type": "object",
"properties": {
"href": {
@@ -10488,28 +10540,61 @@
"description"
]
},
"CustomerLinksResponse": {
"CustomerSidebarSection": {
"type": "object",
"properties": {
"links": {
"title": {
"type": "object"
},
"items": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
"oneOf": [
{
"$ref": "#/components/schemas/CustomerSidebarMenuItem"
},
{
"$ref": "#/components/schemas/CustomerSidebarLinkItem"
}
]
}
}
},
"required": [
"links"
"title",
"items"
]
},
"CustomerLinksConfig": {
"type": "object",
"properties": {
"home": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLinkItem"
}
},
"sidebar": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerSidebarSection"
}
}
}
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"$ref": "#/components/schemas/CustomerLinksConfig"
}
}
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
"$ref": "#/components/schemas/CustomerLinksConfig"
}
},
"required": [
+79 -7
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.28",
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -47,6 +47,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
@@ -1744,10 +1745,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.28",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
"integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
"license": "ISC",
"version": "3.38.0-beta.30",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz",
"integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
@@ -2928,6 +2928,20 @@
"node": ">= 0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"engines": [
"node >= 0.8"
],
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -3097,6 +3111,24 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/@nestjs/platform-express/node_modules/multer": {
"version": "1.4.4-lts.1",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.0.0",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.4",
"object-assign": "^4.1.1",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3121,6 +3153,25 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3195,6 +3246,19 @@
"node": ">= 0.8"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/tslib": {
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
@@ -6509,7 +6573,6 @@
"engines": [
"node >= 6.0"
],
"license": "MIT",
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
@@ -9128,6 +9191,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
},
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -10725,7 +10793,6 @@
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
"license": "MIT",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.6.0",
@@ -11728,6 +11795,11 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/process-warning": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
+2 -1
View File
@@ -35,7 +35,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.28",
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -65,6 +65,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
+1
View File
@@ -111,3 +111,4 @@ function configureSwagger(app: INestApplication) {
);
}
bootstrap();
+2 -1
View File
@@ -478,6 +478,7 @@ export class AuthController {
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
this.logger.info(JSON.stringify(req.headers));
// Check for API key header first
const apiKey = req.get('X-Api-key');
@@ -502,7 +503,7 @@ export class AuthController {
const accessToken = req.cookies['ddf-auth'];
const refreshToken = req.cookies['ddf-refresh-auth'];
const userId = req.cookies['ddf-user-id'];
const resourceHost = req.headers["host"]
const resourceHost = req.headers["x-original-url"] as string || "" ;
const hasUserSession = Boolean(accessToken) && Boolean(userId);
this.logger.info('Has User Session: ' + hasUserSession);
+58
View File
@@ -734,6 +734,64 @@ class CatalogService implements OnModuleInit {
}
}
async renameTableOnNimbus(
nimbusUrl: string,
nimbusId: number,
changes: { table_name?: string; table_schema?: string; display_name?: string },
): Promise<void> {
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
await axios.patch(endpoint, changes);
}
async renameColumnMetadataOnNimbus(
nimbusUrl: string,
databaseName: string,
oldTableName: string,
oldTableSchema: string,
newTableName: string,
newTableSchema: string,
): Promise<void> {
const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
const { data: columns } = await axios.get(listEndpoint);
const filtered = Array.isArray(columns) ? columns : [];
for (const column of filtered) {
const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
await axios.patch(patchEndpoint, {
table_name: newTableName,
table_schema: newTableSchema,
});
}
this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
}
async renameDataPreviewOnNimbus(
nimbusUrl: string,
databaseName: string,
oldTableName: string,
oldTableSchema: string,
newTableName: string,
newTableSchema: string,
): Promise<void> {
const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
const { data: previews } = await axios.get(listEndpoint);
const filtered = Array.isArray(previews) ? previews : [];
for (const preview of filtered) {
const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
await axios.patch(patchEndpoint, {
table_name: newTableName,
table_schema: newTableSchema,
});
}
this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
+10 -10
View File
@@ -9,12 +9,12 @@ import {
} from '@nestjs/common';
import { firstValueFrom, lastValueFrom } from 'rxjs';
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { CustomerLinksConfig } from './dtos/customers';
import ErrorCodes from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import {
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
)
}
async getLinks(customerId: string) {
async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
try {
const result = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId }),
this.customerService.CustomerGetLinks({ customerId }),
);
return result.customer?.links || [];
return (result.links as CustomerLinksConfig) || null;
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
}
}
async setLinks(customerId: string, links: Link[]) {
async setLinks(customerId: string, links: CustomerLinksConfig) {
if (!customerId || !links) {
throw new HttpException(null, HttpStatus.BAD_REQUEST);
}
try {
return await firstValueFrom(
this.customerService.CustomerUpdate({
id: customerId,
links,
} as CustomerUpdateRequest),
this.customerService.CustomerSetLinks({
customerId,
links: links as CustomerSetLinksRequest['links'],
}),
);
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+52 -9
View File
@@ -1,7 +1,6 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class CustomerLink implements Link {
export class CustomerLinkItem {
@ApiProperty()
href: string;
@ApiProperty()
@@ -9,15 +8,59 @@ export class CustomerLink implements Link {
@ApiProperty()
description: string;
@ApiPropertyOptional()
iconSrc: string;
iconSrc?: string;
}
export class CustomerSidebarLinkItem {
@ApiProperty()
type: 'link';
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiProperty()
link: string;
@ApiPropertyOptional()
icon?: string;
}
export class CustomerSidebarMenuItem {
@ApiProperty()
type: 'menu';
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiPropertyOptional()
icon?: string;
@ApiProperty({ type: [CustomerSidebarLinkItem] })
items: CustomerSidebarLinkItem[];
}
export class CustomerSidebarSection {
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiProperty({
type: 'array',
items: {
oneOf: [
{ $ref: '#/components/schemas/CustomerSidebarMenuItem' },
{ $ref: '#/components/schemas/CustomerSidebarLinkItem' },
],
},
})
items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
}
export class CustomerLinksConfig {
@ApiPropertyOptional({ type: [CustomerLinkItem] })
home?: CustomerLinkItem[];
@ApiPropertyOptional({ type: [CustomerSidebarSection] })
sidebar?: CustomerSidebarSection[];
}
export class CustomerLinkRequest {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
@ApiProperty({ type: CustomerLinksConfig })
links: CustomerLinksConfig;
}
export class CustomerLinksResponse {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
}
@ApiPropertyOptional({ type: CustomerLinksConfig })
links?: CustomerLinksConfig;
}
+5 -3
View File
@@ -11,17 +11,19 @@ export class TableColumns {
name: string;
@ApiProperty()
columns: string[];
@ApiProperty()
@ApiPropertyOptional({ type: [Column] })
references: Column[];
@ApiProperty()
identifier_columns: string[];
@ApiProperty()
destination: Record<'raw' | 'qualify', {
table_name: string;
table_schema: string;
}> | null;
@ApiProperty()
type: string;
@ApiPropertyOptional({ type: [String] })
identifier_columns?: string[];
@ApiPropertyOptional({ type: Column })
reference_column?: Column;
}
export class AvailableEntity {
@ApiProperty()
+5
View File
@@ -164,6 +164,11 @@ export class InputsService {
const inputCreateGenericRequest: InputCreateGenericRequest = {
input: {
...body,
tables: (body.tables || []).map((table) => ({
...table,
identifier_columns: table.identifier_columns || [],
reference_column: table.reference_column || table.references?.[0],
})),
},
info,
};
@@ -10,6 +10,7 @@ import {
Query,
Inject,
BadRequestException,
HttpException,
} from '@nestjs/common';
import { ApiTags, ApiOperation } from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -25,6 +26,8 @@ import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
import { CatalogService } from '../catalog/catalog.service';
import { PackTheMetadata } from '../../utils/PackTheMetadata';
import { ValidationTableDTO } from './platform-api.dto';
@@ -35,6 +38,11 @@ type ValidateTablesDTO = {
}>
}
type RenameTablesBody = {
raw?: { table_name: string; table_schema: string };
qualify?: { table_name: string; table_schema: string };
}
@ApiTags('Platform API')
@Controller('platform')
export class PlatformApiController {
@@ -45,6 +53,7 @@ export class PlatformApiController {
private readonly elasticsearchService: ElasticsearchService,
private readonly dynamoDBService: DynamoDBService,
private readonly customersService: CustomersService,
private readonly catalogService: CatalogService,
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
@@ -76,6 +85,23 @@ export class PlatformApiController {
return jobId?.replace(/-/g, '_') || '';
}
private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise<any> {
const connectorTypes = ['jdbc', 'singer', 's3'];
for (const type of connectorTypes) {
try {
const job = await this.platformApiService.proxy(
'GET',
`/jobs/${type}/${normalizedJobId}`,
user,
);
return job;
} catch (error) {
// Continue to next connector type
}
}
throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404);
}
/**
* Extract the pipeline ID (base UUID) from a job ID.
* Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1").
@@ -953,6 +979,192 @@ export class PlatformApiController {
return result;
}
@Post('jobs/:jobId/rename-tables')
@ApiOperation({ summary: 'Rename job output tables and sync to catalog' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async renameJobTables(
@Param('jobId') jobId: string,
@Body() body: RenameTablesBody,
@User() user: RequestUser,
) {
const normalizedJobId = this.normalizeJobId(jobId);
const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user);
const result = await this.platformApiService.proxy(
'POST',
`/jobs/${normalizedJobId}/rename-tables`,
user,
body,
);
try {
await this.syncTableRenameToCatalog(jobId, body, currentJob, user);
} catch (error) {
this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message });
const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {});
if (reverseBody) {
try {
await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody);
this.logger.info('Snowflake rename rolled back', { jobId });
} catch (rollbackError) {
this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message });
}
}
throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500);
}
return result;
}
private buildSnowflakeRollbackBody(
body: RenameTablesBody,
outputConfig: any,
): RenameTablesBody | null {
const reverse: RenameTablesBody = {};
if (body.raw) {
const nested = outputConfig.raw;
const oldTableName = nested?.table_name || outputConfig.table_name;
const oldTableSchema = nested?.table_schema || 'PUBLIC';
if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema };
}
if (body.qualify) {
const nested = outputConfig.qualify;
if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' };
}
return Object.keys(reverse).length > 0 ? reverse : null;
}
/**
* Sync table rename to Elasticsearch and Nimbus.
*
* For each target (raw, qualify):
* 1. Resolve old table name from output_config
* 2. Find the ES data asset by pipeline + table + schema
* 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview
* 4. If any step fails, rollback all completed steps for that target
*/
private async syncTableRenameToCatalog(
jobId: string,
body: RenameTablesBody,
currentJob: any,
user: RequestUser,
): Promise<void> {
const pipelineId = this.extractPipelineIdFromJobId(jobId);
const outputConfig = currentJob.output_config || {};
const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } });
const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`;
const targets = this.buildRenameTargets(body, outputConfig);
for (const { key, oldTableName, oldTableSchema, newValues } of targets) {
const rollbackSteps: Array<() => Promise<void>> = [];
try {
const dataAsset = await this.elasticsearchService.findDataAssetByTable(
user.customer_name, oldTableName, oldTableSchema,
);
if (!dataAsset) {
this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema });
continue;
}
const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset;
const oldValues = { table_name: oldTableName, table_schema: oldTableSchema };
// ES update
const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name };
await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields);
rollbackSteps.push(() => this.elasticsearchService.updateDataAsset(
user.customer_name, esAssetId,
{ name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName },
));
const newTableNameUpper = newValues.table_name.toUpperCase();
const newTableSchemaUpper = newValues.table_schema.toUpperCase();
const oldTableNameUpper = oldTableName.toUpperCase();
const oldTableSchemaUpper = oldTableSchema.toUpperCase();
// Nimbus table-metadata
if (nimbusId) {
await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper });
rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper }));
}
// Nimbus column-metadata
await this.catalogService.renameColumnMetadataOnNimbus(
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
);
rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus(
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
));
// Nimbus data-preview
await this.catalogService.renameDataPreviewOnNimbus(
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
);
rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus(
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
));
this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name });
} catch (error) {
this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message });
await this.executeRollback(rollbackSteps, key, jobId);
throw error;
}
}
}
private buildRenameTargets(
body: RenameTablesBody,
outputConfig: any,
): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> {
const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' };
const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = [];
for (const key of ['raw', 'qualify'] as const) {
if (!body[key]) continue;
const nested = outputConfig[key];
// qualify: only sync if output_config.qualify already exists
if (key === 'qualify' && !nested?.table_name) continue;
const oldTableName = nested?.table_name || outputConfig.table_name;
if (!oldTableName) continue;
targets.push({
key,
oldTableName,
oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key],
newValues: body[key],
});
}
return targets;
}
private async executeRollback(
steps: Array<() => Promise<void>>,
targetKey: string,
jobId: string,
): Promise<void> {
for (const rollback of steps.reverse()) {
try {
await rollback();
} catch (error) {
this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message });
}
}
}
@Get('jobs/jdbc/configs/allowed_datatypes')
@ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
@@ -7,9 +7,10 @@ import { PlatformApiService } from './platform-api.service';
import { ElasticsearchModule } from '../../services/elasticsearch';
import { DynamoDBModule } from '../../services/dynamodb';
import { CustomersModule } from '../customers/customers.module';
import { CatalogModule } from '../catalog/catalog.module';
@Module({
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule],
controllers: [PlatformApiController],
providers: [PlatformApiService, DadosferaLogger],
exports: [PlatformApiService],
@@ -89,6 +89,12 @@ export class PlatformApiService {
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
this.logger.error('Platform API upstream error', {
status: response.status,
data: response.data,
path,
method: method.toUpperCase(),
});
throw new HttpException(response.data, response.status);
}
@@ -358,6 +358,81 @@ export class ElasticsearchService {
}
}
private getDataAssetIndex(customerName: string): string {
return `${customerName}_data_assets_catalog`;
}
async findDataAssetByTable(
customerName: string,
tableName: string,
tableSchema: string,
): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> {
const index = this.getDataAssetIndex(customerName);
this.logger.info('Elasticsearch: Searching data asset', {
index,
tableName,
tableSchema,
});
try {
const response = await this.client.post(`/${index}/_search`, {
query: {
bool: {
must: [
{ term: { 'table_name.keyword': tableName.toUpperCase() } },
{ term: { 'table_schema.keyword': tableSchema.toUpperCase() } },
],
},
},
size: 1,
});
const hits = response.data.hits?.hits || [];
if (hits.length === 0) {
this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index });
return null;
}
return { ...hits[0]._source, _es_id: hits[0]._id };
} catch (error) {
this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index });
throw error;
}
}
async updateDataAsset(
customerName: string,
assetId: string,
updates: Record<string, any>,
): Promise<any> {
const index = this.getDataAssetIndex(customerName);
this.logger.info('Elasticsearch: Updating data asset', {
index,
assetId,
fields: Object.keys(updates),
});
try {
const response = await this.client.post(
`/${index}/_update/${assetId}`,
{ doc: updates },
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Data asset updated', {
assetId,
result: response.data.result,
});
return response.data;
} catch (error) {
this.handleError('updateDataAsset', error, { assetId, index });
throw error;
}
}
private handleError(
operation: string,
error: any,