mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-01 12:18:15 +00:00
Compare commits
71
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
049eca9700 | ||
|
|
3bcd78bb32 | ||
|
|
f603b64679 | ||
|
|
b640465624 | ||
|
|
98282471a9 | ||
|
|
8983430889 | ||
|
|
d3aeca12cd | ||
|
|
ce619942a3 | ||
|
|
71b3f278a5 | ||
|
|
c92542ed90 | ||
|
|
69a9d78642 | ||
|
|
bf4f3cfd8c | ||
|
|
01afa69fcb | ||
|
|
80f3913fd2 | ||
|
|
10293ad6a9 | ||
|
|
e4d0c9c3e6 | ||
|
|
21c57e5620 | ||
|
|
959210e354 | ||
|
|
f10953e949 | ||
|
|
13903b9bb9 | ||
|
|
2e3a13d421 | ||
|
|
fa17fc3001 | ||
|
|
b7171556b8 | ||
|
|
254a638392 | ||
|
|
74b3bd6b46 | ||
|
|
9a29ef5401 | ||
|
|
3f21faaa66 | ||
|
|
bf19d29a1d | ||
|
|
6523f707e3 | ||
|
|
8b0bf84d34 | ||
|
|
9ef4c51ba1 | ||
|
|
19521489fa | ||
|
|
2ce9aad005 | ||
|
|
051fb6e4dd | ||
|
|
2125884c6c | ||
|
|
e3099aa2b2 | ||
|
|
f4c9226ef9 | ||
|
|
12c61d9b5d | ||
|
|
ff3999a6aa | ||
|
|
209470482a | ||
|
|
99c2a9ecf5 | ||
|
|
c0f75d241f | ||
|
|
1e0fb78dff | ||
|
|
c26194554c | ||
|
|
b70d37423d | ||
|
|
3bcbba9581 | ||
|
|
6f9c967c96 | ||
|
|
b8bdc5beea | ||
|
|
269f70b309 | ||
|
|
93f452ae05 | ||
|
|
e39378229f | ||
|
|
61a4f724ef | ||
|
|
38a9e21f5f | ||
|
|
d25bfd147c | ||
|
|
6c57bac235 | ||
|
|
cf8eed35a3 | ||
|
|
55fc85c544 | ||
|
|
72ed637640 | ||
|
|
890364f597 | ||
|
|
30eec733b2 | ||
|
|
30a41ba144 | ||
|
|
2dc032e7e7 | ||
|
|
7f5981731f | ||
|
|
66309c7bbe | ||
|
|
15048eaf8a | ||
|
|
0e169a3cbc | ||
|
|
b5d933eaf3 | ||
|
|
6fa9bf861a | ||
|
|
e08734c97f | ||
|
|
54b75ce11b | ||
|
|
85234fe0dd |
@@ -4,6 +4,7 @@
|
||||
|
||||
# Maestro
|
||||
|
||||
|
||||
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
|
||||
|
||||
```mermaid
|
||||
|
||||
+1455
-1290
File diff suppressed because it is too large
Load Diff
Generated
+79
-6
@@ -17,7 +17,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
|
||||
"@dadosfera/protospack-v2": "3.40.0-beta.1",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -47,6 +47,7 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
@@ -1744,9 +1745,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.38.0-beta.26",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz",
|
||||
"integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==",
|
||||
"version": "3.40.0-beta.1",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.1.tgz",
|
||||
"integrity": "sha512-Oa0AgYDxo9JT/yQ+sGsgCBe1dJZX5ER0kmtjqlzqSg4pGnx1achQOlWOzv23/u54yUBh0TkvKvwffgI4HkgOew==",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"rxjs": "^7.5.5"
|
||||
@@ -2927,6 +2928,20 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
|
||||
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
|
||||
"engines": [
|
||||
"node >= 0.8"
|
||||
],
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^2.2.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
|
||||
"version": "0.5.4",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
|
||||
@@ -3096,6 +3111,24 @@
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/multer": {
|
||||
"version": "1.4.4-lts.1",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
|
||||
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
|
||||
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.0.0",
|
||||
"concat-stream": "^1.5.2",
|
||||
"mkdirp": "^0.5.4",
|
||||
"object-assign": "^4.1.1",
|
||||
"type-is": "^1.6.4",
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
|
||||
@@ -3120,6 +3153,25 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
|
||||
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
|
||||
"dependencies": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
@@ -3194,6 +3246,19 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"dependencies": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/tslib": {
|
||||
"version": "2.5.3",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
|
||||
@@ -6508,7 +6573,6 @@
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
@@ -9127,6 +9191,11 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
|
||||
},
|
||||
"node_modules/isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
@@ -10724,7 +10793,6 @@
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
|
||||
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.6.0",
|
||||
@@ -11727,6 +11795,11 @@
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"node_modules/process-warning": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
|
||||
|
||||
+3
-3
@@ -10,7 +10,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact",
|
||||
"prebuild": "rimraf dist",
|
||||
"build": "nest build",
|
||||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
||||
@@ -35,7 +35,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
|
||||
"@dadosfera/protospack-v2": "3.40.0-beta.1",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -65,6 +65,7 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
@@ -80,7 +81,6 @@
|
||||
"swagger-ui-express": "^4.6.3"
|
||||
},
|
||||
"overrides": {
|
||||
"multer": "2.0.2",
|
||||
"form-data": "^4.0.4",
|
||||
"body-parser": "^1.20.3",
|
||||
"cross-spawn": "^7.0.5",
|
||||
|
||||
@@ -111,3 +111,4 @@ function configureSwagger(app: INestApplication) {
|
||||
);
|
||||
}
|
||||
bootstrap();
|
||||
|
||||
|
||||
@@ -478,6 +478,7 @@ export class AuthController {
|
||||
@Get('me')
|
||||
async getMe(@Req() req: Request, @Res() res: Response) {
|
||||
this.logger.info('GET /auth/me ')
|
||||
this.logger.info(JSON.stringify(req.headers));
|
||||
|
||||
// Check for API key header first
|
||||
const apiKey = req.get('X-Api-key');
|
||||
@@ -488,6 +489,7 @@ export class AuthController {
|
||||
const userDto = {
|
||||
id: api_key.user_id,
|
||||
name: api_key.username,
|
||||
email: api_key.username,
|
||||
customer: {
|
||||
id: api_key.customer_id,
|
||||
name: api_key.customer_name,
|
||||
|
||||
@@ -437,7 +437,8 @@ export class AuthClientService implements OnModuleInit {
|
||||
|
||||
const userDto: UserDTO = {
|
||||
id: user.id,
|
||||
name: user.username,
|
||||
name: user.name,
|
||||
email: user.email,
|
||||
jobTitle: user?.jobTitle || null,
|
||||
department: user?.department || null,
|
||||
hierarchy: user?.hierarchy || null,
|
||||
|
||||
@@ -144,6 +144,7 @@ export interface BulkEditResponse {
|
||||
export type UserDTO = {
|
||||
id: string,
|
||||
name: string,
|
||||
email: string,
|
||||
jobTitle?: string,
|
||||
department?: string,
|
||||
hierarchy?: string,
|
||||
|
||||
@@ -734,6 +734,64 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async renameTableOnNimbus(
|
||||
nimbusUrl: string,
|
||||
nimbusId: number,
|
||||
changes: { table_name?: string; table_schema?: string; display_name?: string },
|
||||
): Promise<void> {
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
|
||||
this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
|
||||
await axios.patch(endpoint, changes);
|
||||
}
|
||||
|
||||
async renameColumnMetadataOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
|
||||
const { data: columns } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(columns) ? columns : [];
|
||||
|
||||
for (const column of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
|
||||
}
|
||||
|
||||
async renameDataPreviewOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
|
||||
const { data: previews } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(previews) ? previews : [];
|
||||
|
||||
for (const preview of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
|
||||
}
|
||||
|
||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||
const customer_name_raw = metadata.get('customer_name');
|
||||
|
||||
|
||||
@@ -9,12 +9,12 @@ import {
|
||||
} from '@nestjs/common';
|
||||
|
||||
import { firstValueFrom, lastValueFrom } from 'rxjs';
|
||||
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
import { CustomerLinksConfig } from './dtos/customers';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import {
|
||||
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
|
||||
)
|
||||
}
|
||||
|
||||
async getLinks(customerId: string) {
|
||||
async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
|
||||
try {
|
||||
const result = await lastValueFrom(
|
||||
this.customerService.CustomerFindOneById({ id: customerId }),
|
||||
this.customerService.CustomerGetLinks({ customerId }),
|
||||
);
|
||||
return result.customer?.links || [];
|
||||
return (result.links as CustomerLinksConfig) || null;
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
|
||||
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async setLinks(customerId: string, links: Link[]) {
|
||||
async setLinks(customerId: string, links: CustomerLinksConfig) {
|
||||
if (!customerId || !links) {
|
||||
throw new HttpException(null, HttpStatus.BAD_REQUEST);
|
||||
}
|
||||
|
||||
try {
|
||||
return await firstValueFrom(
|
||||
this.customerService.CustomerUpdate({
|
||||
id: customerId,
|
||||
links,
|
||||
} as CustomerUpdateRequest),
|
||||
this.customerService.CustomerSetLinks({
|
||||
customerId,
|
||||
links: links as CustomerSetLinksRequest['links'],
|
||||
}),
|
||||
);
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
|
||||
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class CustomerLink implements Link {
|
||||
export class CustomerLinkItem {
|
||||
@ApiProperty()
|
||||
href: string;
|
||||
@ApiProperty()
|
||||
@@ -9,15 +8,59 @@ export class CustomerLink implements Link {
|
||||
@ApiProperty()
|
||||
description: string;
|
||||
@ApiPropertyOptional()
|
||||
iconSrc: string;
|
||||
iconSrc?: string;
|
||||
}
|
||||
|
||||
export class CustomerSidebarLinkItem {
|
||||
@ApiProperty()
|
||||
type: 'link';
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiProperty()
|
||||
link: string;
|
||||
@ApiPropertyOptional()
|
||||
icon?: string;
|
||||
}
|
||||
|
||||
export class CustomerSidebarMenuItem {
|
||||
@ApiProperty()
|
||||
type: 'menu';
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiPropertyOptional()
|
||||
icon?: string;
|
||||
@ApiProperty({ type: [CustomerSidebarLinkItem] })
|
||||
items: CustomerSidebarLinkItem[];
|
||||
}
|
||||
|
||||
export class CustomerSidebarSection {
|
||||
@ApiProperty({ type: Object })
|
||||
title: Record<string, string>;
|
||||
@ApiProperty({
|
||||
type: 'array',
|
||||
items: {
|
||||
oneOf: [
|
||||
{ $ref: '#/components/schemas/CustomerSidebarMenuItem' },
|
||||
{ $ref: '#/components/schemas/CustomerSidebarLinkItem' },
|
||||
],
|
||||
},
|
||||
})
|
||||
items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
|
||||
}
|
||||
|
||||
export class CustomerLinksConfig {
|
||||
@ApiPropertyOptional({ type: [CustomerLinkItem] })
|
||||
home?: CustomerLinkItem[];
|
||||
@ApiPropertyOptional({ type: [CustomerSidebarSection] })
|
||||
sidebar?: CustomerSidebarSection[];
|
||||
}
|
||||
|
||||
export class CustomerLinkRequest {
|
||||
@ApiProperty({ type: [CustomerLink] })
|
||||
links: CustomerLink[];
|
||||
@ApiProperty({ type: CustomerLinksConfig })
|
||||
links: CustomerLinksConfig;
|
||||
}
|
||||
|
||||
export class CustomerLinksResponse {
|
||||
@ApiProperty({ type: [CustomerLink] })
|
||||
links: CustomerLink[];
|
||||
}
|
||||
|
||||
@ApiPropertyOptional({ type: CustomerLinksConfig })
|
||||
links?: CustomerLinksConfig;
|
||||
}
|
||||
@@ -11,10 +11,19 @@ export class TableColumns {
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
columns: string[];
|
||||
@ApiProperty()
|
||||
@ApiPropertyOptional({ type: [Column] })
|
||||
references: Column[];
|
||||
@ApiProperty()
|
||||
destination: Record<'raw' | 'qualify', {
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
}> | null;
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
@ApiPropertyOptional({ type: [String] })
|
||||
identifier_columns?: string[];
|
||||
@ApiPropertyOptional({ type: Column })
|
||||
reference_column?: Column;
|
||||
}
|
||||
export class AvailableEntity {
|
||||
@ApiProperty()
|
||||
|
||||
@@ -99,6 +99,7 @@ export class InputsController {
|
||||
customer: info.customer,
|
||||
});
|
||||
|
||||
this.logger.info(JSON.stringify(body))
|
||||
const response = await this.inputService.create({ body, info });
|
||||
|
||||
return response;
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
InputCreateGenericRequest,
|
||||
InputCreateS3Request,
|
||||
InputNewCreateRequest,
|
||||
InputUpdateResponse,
|
||||
TestConnectionRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
@@ -71,8 +72,8 @@ export class InputsService {
|
||||
objectCamelToSnake(createInputResponse);
|
||||
return createInputResponse;
|
||||
},
|
||||
update: async (updateInputDTO: UpdateInputRequest) => {
|
||||
this.logger.info('InputClientService - Update');
|
||||
update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
|
||||
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
|
||||
const updateInputResponse = await lastValueFrom(
|
||||
this.inputWriteService.InputUpdate(updateInputDTO),
|
||||
);
|
||||
@@ -164,6 +165,11 @@ export class InputsService {
|
||||
const inputCreateGenericRequest: InputCreateGenericRequest = {
|
||||
input: {
|
||||
...body,
|
||||
tables: (body.tables || []).map((table) => ({
|
||||
...table,
|
||||
identifier_columns: table.identifier_columns || [],
|
||||
reference_column: table.reference_column || table.references?.[0],
|
||||
})),
|
||||
},
|
||||
info,
|
||||
};
|
||||
@@ -200,18 +206,24 @@ export class InputsService {
|
||||
}
|
||||
|
||||
async update(id: string, data, info: Info) {
|
||||
this.validateCron({ ...data, info });
|
||||
// this.validateCron({ ...data, info });
|
||||
try {
|
||||
const updateInputResponse: any = await this.OLD_inputClient.update({
|
||||
const {
|
||||
nimbusUpdates,
|
||||
input
|
||||
} = await this.OLD_inputClient.update({
|
||||
id,
|
||||
info,
|
||||
...data,
|
||||
info,
|
||||
});
|
||||
|
||||
updateInputResponse.input = this.adjustInputPayload(
|
||||
updateInputResponse?.input,
|
||||
const updateInputResponse = this.adjustInputPayload(
|
||||
input,
|
||||
);
|
||||
return updateInputResponse;
|
||||
return {
|
||||
input: updateInputResponse,
|
||||
nimbusUpdates
|
||||
};
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export class PipelineInputsDTO {
|
||||
@ApiProperty()
|
||||
tables: Array<{
|
||||
name: string,
|
||||
type: string,
|
||||
|
||||
}>
|
||||
}
|
||||
|
||||
export class IPipelineV2 {
|
||||
@ApiProperty()
|
||||
id: string;
|
||||
@@ -123,3 +132,30 @@ export class PipelineFindAllReq {
|
||||
@ApiPropertyOptional()
|
||||
type?: string | undefined;
|
||||
}
|
||||
|
||||
export interface UpdateTableDTO {
|
||||
name: string;
|
||||
type: string;
|
||||
columns: string[];
|
||||
destinations: {
|
||||
raw: {
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
};
|
||||
qualify: {
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
};
|
||||
};
|
||||
identifier_columns: string[];
|
||||
reference_column: {
|
||||
name: string;
|
||||
type: string;
|
||||
};
|
||||
memory: number;
|
||||
}
|
||||
|
||||
export interface UpdatePlatformInputRequest {
|
||||
cron: string;
|
||||
tables: Array<UpdateTableDTO>;
|
||||
}
|
||||
|
||||
@@ -43,11 +43,15 @@ import {
|
||||
IPipelineV2,
|
||||
IInitUploadCSVFile,
|
||||
PipelineFindAllReq,
|
||||
UpdatePlatformInputRequest,
|
||||
} from './interfaces';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
|
||||
import { TableColumns } from '../inputs/dtos/input.model';
|
||||
import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
|
||||
@ApiTags('PipelinesV2')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@@ -223,6 +227,7 @@ export class PipelinesController {
|
||||
.then((res) => {
|
||||
//{pipeline:{tables: {tables: [], input_id: ''}}}
|
||||
let tables = JSON.parse(res.pipeline.config.tables);
|
||||
const input_id = tables?.input_id;
|
||||
if (tables?.tables) tables = tables.tables;
|
||||
Object.assign(res.pipeline, {
|
||||
transformations: res.pipeline.transformations
|
||||
@@ -231,6 +236,7 @@ export class PipelinesController {
|
||||
config: {
|
||||
cron: res.pipeline.config.cron,
|
||||
tables,
|
||||
input_id
|
||||
},
|
||||
properties: res.pipeline.properties
|
||||
? JSON.parse(res.pipeline.properties)
|
||||
@@ -277,6 +283,46 @@ export class PipelinesController {
|
||||
return response;
|
||||
}
|
||||
|
||||
@Patch('/:pipelineId/inputs/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updatePipelineInput(
|
||||
@Language() language: LanguageEnum,
|
||||
@Body() pipelineInputDTO: UpdatePlatformInputRequest,
|
||||
@Param('id') inputId: string,
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
this.logger.info('PipelinesController - update', { user });
|
||||
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const info: Info = {
|
||||
user_id: user.user_id,
|
||||
customer: user.customer_name,
|
||||
customer_id: user.customer_id,
|
||||
pipeline_id: pipelineId
|
||||
};
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
language,
|
||||
});
|
||||
|
||||
const response = await this.pipelinesClientService.updatePipelineInput(
|
||||
pipelineId,
|
||||
inputId,
|
||||
pipelineInputDTO,
|
||||
info,
|
||||
user,
|
||||
metadata,
|
||||
);
|
||||
|
||||
this.logger.info('PipelinesController - update: OK', { user });
|
||||
return response;
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Put('/:id')
|
||||
@ApiOperation({
|
||||
|
||||
@@ -11,6 +11,9 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe
|
||||
import { ConnectorModule } from '../connector/connector.module';
|
||||
import { InputsModule } from '../inputs/inputs.module';
|
||||
import { TransformationsModule } from '../transformations/transformations.module';
|
||||
import { PlatformApiModule } from '../platform-api/platform-api.module';
|
||||
import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
|
||||
const client = new PipelinesClientConfiguration();
|
||||
|
||||
@@ -21,9 +24,11 @@ const client = new PipelinesClientConfiguration();
|
||||
ConnectorModule,
|
||||
InputsModule,
|
||||
TransformationsModule,
|
||||
PlatformApiModule,
|
||||
NimbusServicesModule
|
||||
],
|
||||
controllers: [PipelinesController],
|
||||
providers: [PipelinesService, DadosferaLogger],
|
||||
providers: [PipelinesService, DadosferaLogger, NimbusService],
|
||||
exports: [PipelinesService],
|
||||
})
|
||||
export class PipelinesV2Module {}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
/* eslint-disable no-async-promise-executor */
|
||||
import {
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
@@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
|
||||
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
import { ICreatePipelineV2Req } from './interfaces';
|
||||
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { ConnectorClientService } from '../connector/client.service';
|
||||
@@ -26,6 +27,11 @@ import { TransformationsService } from '../transformations/transformations.servi
|
||||
import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
import { PlatformApiService } from '../platform-api/platform-api.service';
|
||||
import { Info, NewTable } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { NimbusUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import axios from 'axios';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
|
||||
export class PipelinesService implements OnModuleInit {
|
||||
logger: DadosferaLogger;
|
||||
@@ -39,6 +45,8 @@ export class PipelinesService implements OnModuleInit {
|
||||
private readonly connectorService: ConnectorClientService,
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly transformationsService: TransformationsService,
|
||||
private readonly platformAPI: PlatformApiService,
|
||||
private readonly nimbusService: NimbusService
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
@@ -138,6 +146,7 @@ export class PipelinesService implements OnModuleInit {
|
||||
const findOnePipelineResponse = await lastValueFrom(
|
||||
this.pipelineReadService.PipelineV2FindOne(data, metadata),
|
||||
);
|
||||
console.log('pipeline find one response', findOnePipelineResponse);
|
||||
this.logger.info('Done');
|
||||
|
||||
return findOnePipelineResponse;
|
||||
@@ -339,4 +348,129 @@ export class PipelinesService implements OnModuleInit {
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
|
||||
this.logger.info('InputClientService - Update');
|
||||
|
||||
this.logger.info('Update Dynamo Reference');
|
||||
const pipelineIdFormat = pipelineId.split('-').join('_');
|
||||
console.log("updatePipelineInput", info, updateInputDTO)
|
||||
const updateInputResponse = await this.inputsService.update(
|
||||
inputId,
|
||||
updateInputDTO,
|
||||
info
|
||||
)
|
||||
|
||||
this.logger.info(JSON.stringify(updateInputResponse))
|
||||
await this.updateNimbus(info.customer, updateInputResponse.nimbusUpdates);
|
||||
|
||||
await this.updatePlatformJobs(
|
||||
pipelineIdFormat,
|
||||
updateInputDTO,
|
||||
user
|
||||
);
|
||||
|
||||
return updateInputResponse;
|
||||
}
|
||||
|
||||
private async updateNimbus(customer: string, changes: NimbusUpdate[]) {
|
||||
this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
|
||||
if(!changes || changes.length === 0) return;
|
||||
|
||||
const requests = changes.map(change => {
|
||||
return this.nimbusService.renameTable(customer, change.database, change.old, change.new);
|
||||
})
|
||||
|
||||
const values = await Promise.allSettled(requests);
|
||||
|
||||
const success = values.map(request => request.status === "fulfilled")
|
||||
|
||||
this.logger.info("Updates with succes: " + success.length);
|
||||
}
|
||||
|
||||
async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
|
||||
const jobsUpdated = [];
|
||||
|
||||
for (const [index, table] of updateInputDTO.tables.entries()) {
|
||||
const jobUpdate = {
|
||||
job_id: `${pipelineId}_${index}`,
|
||||
}
|
||||
|
||||
if (table.memory) {
|
||||
jobUpdate["memory"] = {
|
||||
amount: table.memory * 1000
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Updating input reference for table: ' + table.name);
|
||||
let hasUpdateSyncMode = false;
|
||||
|
||||
const jobSyncMode = {}
|
||||
|
||||
if (table.columns) {
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['column_include_list'] = table.columns;
|
||||
}
|
||||
|
||||
if (table.reference_column) {
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['incremental_column_name'] = table.reference_column.name;
|
||||
jobSyncMode['incremental_column_type'] = table.reference_column.type;
|
||||
}
|
||||
|
||||
if (table.identifier_columns) {
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
if (table.type) {
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['target_load_type'] = table.type;
|
||||
|
||||
}
|
||||
|
||||
if(hasUpdateSyncMode) {
|
||||
jobUpdate["sync_mode"] = jobSyncMode;
|
||||
}
|
||||
|
||||
if (Object.keys(table.destinations).length > 1) {
|
||||
let hasChanges = false
|
||||
const jobRenameTables = {
|
||||
raw: {},
|
||||
qualify: {}
|
||||
}
|
||||
|
||||
if (Object.keys(table.destinations.raw).length > 1) {
|
||||
hasChanges = true;
|
||||
jobRenameTables.raw = table.destinations.raw;
|
||||
}
|
||||
|
||||
if (Object.keys(table.destinations.qualify).length > 1) {
|
||||
hasChanges = true;
|
||||
jobRenameTables.qualify = table.destinations.qualify;
|
||||
}
|
||||
|
||||
if (hasChanges) {
|
||||
jobUpdate['rename_tables'] = jobRenameTables;
|
||||
}
|
||||
}
|
||||
|
||||
jobsUpdated.push(jobUpdate);
|
||||
}
|
||||
|
||||
this.logger.info('Request body:' + JSON.stringify({
|
||||
jobs_updated: jobsUpdated
|
||||
}));
|
||||
|
||||
const response = await this.platformAPI.proxy(
|
||||
'PUT',
|
||||
`/pipeline/${pipelineId}/jobs`,
|
||||
user,
|
||||
{
|
||||
job_updates: jobsUpdated
|
||||
}
|
||||
)
|
||||
this.logger.info('Platform api response: ' + JSON.stringify(response));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
Query,
|
||||
Inject,
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
@@ -25,6 +26,9 @@ import { ElasticsearchService } from '../../services/elasticsearch';
|
||||
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
|
||||
import { CustomersService } from '../customers/customers.service';
|
||||
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
||||
import { CatalogService } from '../catalog/catalog.service';
|
||||
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
||||
import { ValidationTableDTO } from './platform-api.dto';
|
||||
|
||||
|
||||
type ValidateTablesDTO = {
|
||||
@@ -34,6 +38,11 @@ type ValidateTablesDTO = {
|
||||
}>
|
||||
}
|
||||
|
||||
type RenameTablesBody = {
|
||||
raw?: { table_name: string; table_schema: string };
|
||||
qualify?: { table_name: string; table_schema: string };
|
||||
}
|
||||
|
||||
@ApiTags('Platform API')
|
||||
@Controller('platform')
|
||||
export class PlatformApiController {
|
||||
@@ -44,6 +53,7 @@ export class PlatformApiController {
|
||||
private readonly elasticsearchService: ElasticsearchService,
|
||||
private readonly dynamoDBService: DynamoDBService,
|
||||
private readonly customersService: CustomersService,
|
||||
private readonly catalogService: CatalogService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
@@ -75,6 +85,23 @@ export class PlatformApiController {
|
||||
return jobId?.replace(/-/g, '_') || '';
|
||||
}
|
||||
|
||||
private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise<any> {
|
||||
const connectorTypes = ['jdbc', 'singer', 's3'];
|
||||
for (const type of connectorTypes) {
|
||||
try {
|
||||
const job = await this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/jobs/${type}/${normalizedJobId}`,
|
||||
user,
|
||||
);
|
||||
return job;
|
||||
} catch (error) {
|
||||
// Continue to next connector type
|
||||
}
|
||||
}
|
||||
throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the pipeline ID (base UUID) from a job ID.
|
||||
* Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1").
|
||||
@@ -728,26 +755,10 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== Catalog ROUTES ====================
|
||||
|
||||
@Get('pipelines/catalog/tables')
|
||||
@ApiOperation({ summary: 'Get all tables available' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getAvailableTables(
|
||||
@User() user: RequestUser,
|
||||
@Query() query: Record<string, string>,
|
||||
) {
|
||||
return this.platformApiService.proxy(
|
||||
'GET',
|
||||
'/catalog/tables',
|
||||
user,
|
||||
undefined,
|
||||
query,
|
||||
);
|
||||
}
|
||||
// ==================== PIPELINE VALIDATION ====================
|
||||
|
||||
@Get('pipelines/catalog/schemas')
|
||||
@ApiOperation({ summary: 'Get all schemas available' })
|
||||
@ApiOperation({ summary: 'Get available schemas' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getAvailableSchemas(
|
||||
@User() user: RequestUser,
|
||||
@@ -755,7 +766,7 @@ export class PlatformApiController {
|
||||
) {
|
||||
return this.platformApiService.proxy(
|
||||
'GET',
|
||||
'/catalog/schemas',
|
||||
`/catalog/schemas`,
|
||||
user,
|
||||
undefined,
|
||||
query,
|
||||
@@ -763,18 +774,18 @@ export class PlatformApiController {
|
||||
}
|
||||
|
||||
@Post('pipelines/catalog/tables/validate')
|
||||
@ApiOperation({ summary: 'Validate tables and schemas' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
@ApiOperation({ summary: 'Validate Table and Schema' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async validateTableAndSchema(
|
||||
@Body() payload: ValidationTableDTO,
|
||||
@User() user: RequestUser,
|
||||
@Query() query: Record<string, string>,
|
||||
@Body() validateTablesDto: ValidateTablesDTO[]
|
||||
) {
|
||||
return this.platformApiService.proxy(
|
||||
'POST',
|
||||
'/catalog/tables/validate',
|
||||
`/catalog/tables/validate`,
|
||||
user,
|
||||
validateTablesDto,
|
||||
payload,
|
||||
query,
|
||||
);
|
||||
}
|
||||
@@ -834,6 +845,24 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Post('pipeline/:pipelineId/pipeline_run/:runId/cancel')
|
||||
@ApiOperation({ summary: 'Cancel a running pipeline run' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async cancelPipelineRun(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('runId') runId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const normalizedRunId = this.normalizePipelineId(runId);
|
||||
|
||||
return this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== JOBS - COLUMN EDITING ROUTES ====================
|
||||
|
||||
@Put('jobs/:jobId/input')
|
||||
@@ -968,6 +997,192 @@ export class PlatformApiController {
|
||||
return result;
|
||||
}
|
||||
|
||||
@Post('jobs/:jobId/rename-tables')
|
||||
@ApiOperation({ summary: 'Rename job output tables and sync to catalog' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async renameJobTables(
|
||||
@Param('jobId') jobId: string,
|
||||
@Body() body: RenameTablesBody,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
|
||||
const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user);
|
||||
|
||||
const result = await this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/jobs/${normalizedJobId}/rename-tables`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.syncTableRenameToCatalog(jobId, body, currentJob, user);
|
||||
} catch (error) {
|
||||
this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message });
|
||||
|
||||
const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {});
|
||||
if (reverseBody) {
|
||||
try {
|
||||
await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody);
|
||||
this.logger.info('Snowflake rename rolled back', { jobId });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message });
|
||||
}
|
||||
}
|
||||
|
||||
throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private buildSnowflakeRollbackBody(
|
||||
body: RenameTablesBody,
|
||||
outputConfig: any,
|
||||
): RenameTablesBody | null {
|
||||
const reverse: RenameTablesBody = {};
|
||||
|
||||
if (body.raw) {
|
||||
const nested = outputConfig.raw;
|
||||
const oldTableName = nested?.table_name || outputConfig.table_name;
|
||||
const oldTableSchema = nested?.table_schema || 'PUBLIC';
|
||||
if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema };
|
||||
}
|
||||
|
||||
if (body.qualify) {
|
||||
const nested = outputConfig.qualify;
|
||||
if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' };
|
||||
}
|
||||
|
||||
return Object.keys(reverse).length > 0 ? reverse : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync table rename to Elasticsearch and Nimbus.
|
||||
*
|
||||
* For each target (raw, qualify):
|
||||
* 1. Resolve old table name from output_config
|
||||
* 2. Find the ES data asset by pipeline + table + schema
|
||||
* 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview
|
||||
* 4. If any step fails, rollback all completed steps for that target
|
||||
*/
|
||||
private async syncTableRenameToCatalog(
|
||||
jobId: string,
|
||||
body: RenameTablesBody,
|
||||
currentJob: any,
|
||||
user: RequestUser,
|
||||
): Promise<void> {
|
||||
const pipelineId = this.extractPipelineIdFromJobId(jobId);
|
||||
const outputConfig = currentJob.output_config || {};
|
||||
const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } });
|
||||
const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`;
|
||||
|
||||
const targets = this.buildRenameTargets(body, outputConfig);
|
||||
|
||||
for (const { key, oldTableName, oldTableSchema, newValues } of targets) {
|
||||
const rollbackSteps: Array<() => Promise<void>> = [];
|
||||
|
||||
try {
|
||||
const dataAsset = await this.elasticsearchService.findDataAssetByTable(
|
||||
user.customer_name, oldTableName, oldTableSchema,
|
||||
);
|
||||
|
||||
if (!dataAsset) {
|
||||
this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema });
|
||||
continue;
|
||||
}
|
||||
|
||||
const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset;
|
||||
const oldValues = { table_name: oldTableName, table_schema: oldTableSchema };
|
||||
|
||||
// ES update
|
||||
const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name };
|
||||
await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields);
|
||||
rollbackSteps.push(() => this.elasticsearchService.updateDataAsset(
|
||||
user.customer_name, esAssetId,
|
||||
{ name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName },
|
||||
));
|
||||
|
||||
const newTableNameUpper = newValues.table_name.toUpperCase();
|
||||
const newTableSchemaUpper = newValues.table_schema.toUpperCase();
|
||||
const oldTableNameUpper = oldTableName.toUpperCase();
|
||||
const oldTableSchemaUpper = oldTableSchema.toUpperCase();
|
||||
|
||||
// Nimbus table-metadata
|
||||
if (nimbusId) {
|
||||
await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper });
|
||||
rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper }));
|
||||
}
|
||||
|
||||
// Nimbus column-metadata
|
||||
await this.catalogService.renameColumnMetadataOnNimbus(
|
||||
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
|
||||
);
|
||||
rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus(
|
||||
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
|
||||
));
|
||||
|
||||
// Nimbus data-preview
|
||||
await this.catalogService.renameDataPreviewOnNimbus(
|
||||
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
|
||||
);
|
||||
rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus(
|
||||
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
|
||||
));
|
||||
|
||||
this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name });
|
||||
} catch (error) {
|
||||
this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message });
|
||||
await this.executeRollback(rollbackSteps, key, jobId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private buildRenameTargets(
|
||||
body: RenameTablesBody,
|
||||
outputConfig: any,
|
||||
): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> {
|
||||
const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' };
|
||||
const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = [];
|
||||
|
||||
for (const key of ['raw', 'qualify'] as const) {
|
||||
if (!body[key]) continue;
|
||||
|
||||
const nested = outputConfig[key];
|
||||
|
||||
// qualify: only sync if output_config.qualify already exists
|
||||
if (key === 'qualify' && !nested?.table_name) continue;
|
||||
|
||||
const oldTableName = nested?.table_name || outputConfig.table_name;
|
||||
if (!oldTableName) continue;
|
||||
|
||||
targets.push({
|
||||
key,
|
||||
oldTableName,
|
||||
oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key],
|
||||
newValues: body[key],
|
||||
});
|
||||
}
|
||||
|
||||
return targets;
|
||||
}
|
||||
|
||||
private async executeRollback(
|
||||
steps: Array<() => Promise<void>>,
|
||||
targetKey: string,
|
||||
jobId: string,
|
||||
): Promise<void> {
|
||||
for (const rollback of steps.reverse()) {
|
||||
try {
|
||||
await rollback();
|
||||
} catch (error) {
|
||||
this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Get('jobs/jdbc/configs/allowed_datatypes')
|
||||
@ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { ApiProperty } from "@nestjs/swagger";
|
||||
|
||||
export class ValidationTableDTO {
|
||||
@ApiProperty()
|
||||
tables: Array<{
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
}>
|
||||
}
|
||||
@@ -7,9 +7,10 @@ import { PlatformApiService } from './platform-api.service';
|
||||
import { ElasticsearchModule } from '../../services/elasticsearch';
|
||||
import { DynamoDBModule } from '../../services/dynamodb';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
import { CatalogModule } from '../catalog/catalog.module';
|
||||
|
||||
@Module({
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule],
|
||||
controllers: [PlatformApiController],
|
||||
providers: [PlatformApiService, DadosferaLogger],
|
||||
exports: [PlatformApiService],
|
||||
|
||||
@@ -89,6 +89,12 @@ export class PlatformApiService {
|
||||
|
||||
// Propagate non-2xx responses as HttpExceptions
|
||||
if (response.status >= 400) {
|
||||
this.logger.error('Platform API upstream error' + JSON.stringify({
|
||||
status: response.status,
|
||||
data: response.data,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
}));
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
@@ -101,6 +107,8 @@ export class PlatformApiService {
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
this.logger.error(error)
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -358,6 +358,81 @@ export class ElasticsearchService {
|
||||
}
|
||||
}
|
||||
|
||||
private getDataAssetIndex(customerName: string): string {
|
||||
return `${customerName}_data_assets_catalog`;
|
||||
}
|
||||
|
||||
async findDataAssetByTable(
|
||||
customerName: string,
|
||||
tableName: string,
|
||||
tableSchema: string,
|
||||
): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> {
|
||||
const index = this.getDataAssetIndex(customerName);
|
||||
|
||||
this.logger.info('Elasticsearch: Searching data asset', {
|
||||
index,
|
||||
tableName,
|
||||
tableSchema,
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.post(`/${index}/_search`, {
|
||||
query: {
|
||||
bool: {
|
||||
must: [
|
||||
{ term: { 'table_name.keyword': tableName.toUpperCase() } },
|
||||
{ term: { 'table_schema.keyword': tableSchema.toUpperCase() } },
|
||||
],
|
||||
},
|
||||
},
|
||||
size: 1,
|
||||
});
|
||||
|
||||
const hits = response.data.hits?.hits || [];
|
||||
if (hits.length === 0) {
|
||||
this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index });
|
||||
return null;
|
||||
}
|
||||
|
||||
return { ...hits[0]._source, _es_id: hits[0]._id };
|
||||
} catch (error) {
|
||||
this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async updateDataAsset(
|
||||
customerName: string,
|
||||
assetId: string,
|
||||
updates: Record<string, any>,
|
||||
): Promise<any> {
|
||||
const index = this.getDataAssetIndex(customerName);
|
||||
|
||||
this.logger.info('Elasticsearch: Updating data asset', {
|
||||
index,
|
||||
assetId,
|
||||
fields: Object.keys(updates),
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.post(
|
||||
`/${index}/_update/${assetId}`,
|
||||
{ doc: updates },
|
||||
{ params: { refresh: 'wait_for' } },
|
||||
);
|
||||
|
||||
this.logger.info('Elasticsearch: Data asset updated', {
|
||||
assetId,
|
||||
result: response.data.result,
|
||||
});
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.handleError('updateDataAsset', error, { assetId, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private handleError(
|
||||
operation: string,
|
||||
error: any,
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { NimbusService } from "./nimbus.service";
|
||||
import DadosferaLogger from "@dadosfera/dadosfera-logs";
|
||||
|
||||
@Module({
|
||||
providers: [NimbusService, DadosferaLogger],
|
||||
exports: [NimbusService],
|
||||
})
|
||||
export class NimbusServicesModule {}
|
||||
@@ -0,0 +1,56 @@
|
||||
import DadosferaLogger from "@dadosfera/dadosfera-logs";
|
||||
import { Inject, Injectable } from "@nestjs/common";
|
||||
import axios from "axios";
|
||||
|
||||
type TableUpdate = {
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class NimbusService {
|
||||
private logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
private buildUrl(customerName: string) {
|
||||
if (process.env.ENV === 'prd') {
|
||||
return `https://nimbus-${customerName}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
return `https://nimbus-${customerName}.${process.env.ENV.replace(
|
||||
'local',
|
||||
'stg',
|
||||
)}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
async renameTable(customerName: string, database: string, old: TableUpdate, update: TableUpdate) {
|
||||
const nimbusUrl = this.buildUrl(customerName);
|
||||
|
||||
const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`;
|
||||
|
||||
try {
|
||||
this.logger.info("Request for PATCH" + nimbusUrl + path);
|
||||
this.logger.info("Payload: " + JSON.stringify(update));
|
||||
const { data } = await axios.patch(nimbusUrl + path, {
|
||||
table_name: update.table_name,
|
||||
table_schema: update.table_schema
|
||||
})
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
return {
|
||||
message: error.message,
|
||||
database,
|
||||
old,
|
||||
update
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user