Compare commits

...
Author SHA1 Message Date
Marcos Rodrigues Silva 13903b9bb9 Merge pull request #463 from dadosfera/fix/user-from-me-endpoint
FIX: send correct name and email from user payload
2026-03-19 14:36:58 -03:00
marcos-silva-rodrigues 2e3a13d421 FIX: send correct name and email from user payload 2026-03-19 14:35:14 -03:00
vinicius gadea fa17fc3001 Merge pull request #462 from dadosfera/feat/sidebar-menu
FIX: add multer@2.0.2 and update package-lock.json — missing transiti…
2026-03-19 14:00:58 -03:00
viniciusgadeaandClaude Sonnet 4.6 b7171556b8 FIX: add multer@2.0.2 and update package-lock.json — missing transitive dep of protospack-v2@3.38.0-beta.30
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-19 13:58:02 -03:00
vinicius gadea 254a638392 Merge pull request #457 from dadosfera/feat/sidebar-menu
Feat/sidebar menu
2026-03-19 10:41:12 -03:00
viniciusgadea 74b3bd6b46 FEAT: update CustomerSidebarSection and related DTOs to use object type for title; update protospack-v2 version; update from id to customerId 2026-03-19 08:14:56 -03:00
Marcos Rodrigues Silva 9a29ef5401 Merge pull request #461 from dadosfera/feat/qualify
FEAT: using batch route to update pipeline
2026-03-18 15:05:36 -03:00
marcos-silva-rodrigues 3f21faaa66 FEAT: using batch route to update pipeline 2026-03-18 11:25:55 -03:00
viniciusgadea bf19d29a1d FIX: docsfera.json 2026-03-17 15:38:35 -03:00
viniciusgadeaandClaude Sonnet 4.6 6523f707e3 FIX: remove duplicate identifier_columns declaration in TableColumns DTO
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-17 15:37:08 -03:00
viniciusgadeaandClaude Sonnet 4.6 8b0bf84d34 FIX: resolve merge conflict in package.json for protospack-v2 version
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-17 15:34:12 -03:00
viniciusgadea 9ef4c51ba1 FEAT: update CustomerSidebarSection to support both menu and link items 2026-03-17 15:25:55 -03:00
vinicius gadea 19521489fa Merge branch 'beta' into feat/sidebar-menu 2026-03-17 10:22:37 -03:00
Rafael Santana 2ce9aad005 Merge pull request #460 from dadosfera/force-deploy
FIX: uppercase table_name and table_schema in Nimbus rename calls
2026-03-16 10:42:41 -03:00
Rafael Santana 2125884c6c Merge pull request #459 from dadosfera/force-deploy
FIX: uppercase table_name and table_schema in ES lookup
2026-03-16 10:19:02 -03:00
Rafael Santana f4c9226ef9 Merge pull request #458 from dadosfera/force-deploy
FIX: rename-tables proxy path and ES lookup
2026-03-13 18:29:53 -03:00
viniciusgadea ff3999a6aa FEAT: update customer links handling and DTOs for improved structure sidebar. update protospack-v2 2026-03-13 17:35:56 -03:00
Rafael Santana 209470482a Merge pull request #456 from dadosfera/force-deploy
UPDATE: force deployment of maestro
2026-03-12 18:03:19 -03:00
Marcos Rodrigues Silva c26194554c Merge pull request #455 from dadosfera/fix/header-validation
Fix/header validation
2026-03-11 11:58:23 -03:00
12 changed files with 323 additions and 144 deletions
+103 -18
View File
@@ -4747,6 +4747,42 @@
]
}
},
"/platform/jobs/{jobId}/rename-tables": {
"post": {
"operationId": "PlatformApiController_renameJobTables",
"summary": "Rename job output tables and sync to catalog",
"parameters": [
{
"name": "jobId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/jdbc/configs/allowed_datatypes": {
"get": {
"operationId": "PlatformApiController_getJdbcAllowedDatatypes",
@@ -10125,6 +10161,21 @@
"entities"
]
},
"Column": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string"
}
},
"required": [
"name",
"type"
]
},
"TableColumns": {
"type": "object",
"properties": {
@@ -10140,13 +10191,7 @@
"references": {
"type": "array",
"items": {
"type": "string"
}
},
"identifier_columns": {
"type": "array",
"items": {
"type": "string"
"$ref": "#/components/schemas/Column"
}
},
"destination": {
@@ -10154,13 +10199,20 @@
},
"type": {
"type": "string"
},
"identifier_columns": {
"type": "array",
"items": {
"type": "string"
}
},
"reference_column": {
"$ref": "#/components/schemas/Column"
}
},
"required": [
"name",
"columns",
"references",
"identifier_columns",
"destination",
"type"
]
@@ -10466,7 +10518,7 @@
"enabled"
]
},
"CustomerLink": {
"CustomerLinkItem": {
"type": "object",
"properties": {
"href": {
@@ -10488,28 +10540,61 @@
"description"
]
},
"CustomerLinksResponse": {
"CustomerSidebarSection": {
"type": "object",
"properties": {
"links": {
"title": {
"type": "object"
},
"items": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
"oneOf": [
{
"$ref": "#/components/schemas/CustomerSidebarMenuItem"
},
{
"$ref": "#/components/schemas/CustomerSidebarLinkItem"
}
]
}
}
},
"required": [
"links"
"title",
"items"
]
},
"CustomerLinksConfig": {
"type": "object",
"properties": {
"home": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLinkItem"
}
},
"sidebar": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerSidebarSection"
}
}
}
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"$ref": "#/components/schemas/CustomerLinksConfig"
}
}
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
"$ref": "#/components/schemas/CustomerLinksConfig"
}
},
"required": [
+79 -7
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.28",
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -47,6 +47,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
@@ -1744,10 +1745,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.28",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
"integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
"license": "ISC",
"version": "3.38.0-beta.30",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz",
"integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
@@ -2928,6 +2928,20 @@
"node": ">= 0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"engines": [
"node >= 0.8"
],
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -3097,6 +3111,24 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/@nestjs/platform-express/node_modules/multer": {
"version": "1.4.4-lts.1",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.0.0",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.4",
"object-assign": "^4.1.1",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3121,6 +3153,25 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3195,6 +3246,19 @@
"node": ">= 0.8"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/tslib": {
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
@@ -6509,7 +6573,6 @@
"engines": [
"node >= 6.0"
],
"license": "MIT",
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
@@ -9128,6 +9191,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
},
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -10725,7 +10793,6 @@
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
"license": "MIT",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.6.0",
@@ -11728,6 +11795,11 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/process-warning": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
+2 -1
View File
@@ -35,7 +35,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.28",
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -65,6 +65,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
+1
View File
@@ -489,6 +489,7 @@ export class AuthController {
const userDto = {
id: api_key.user_id,
name: api_key.username,
email: api_key.username,
customer: {
id: api_key.customer_id,
name: api_key.customer_name,
+2 -1
View File
@@ -437,7 +437,8 @@ export class AuthClientService implements OnModuleInit {
const userDto: UserDTO = {
id: user.id,
name: user.username,
name: user.name,
email: user.email,
jobTitle: user?.jobTitle || null,
department: user?.department || null,
hierarchy: user?.hierarchy || null,
+1
View File
@@ -144,6 +144,7 @@ export interface BulkEditResponse {
export type UserDTO = {
id: string,
name: string,
email: string,
jobTitle?: string,
department?: string,
hierarchy?: string,
+10 -10
View File
@@ -9,12 +9,12 @@ import {
} from '@nestjs/common';
import { firstValueFrom, lastValueFrom } from 'rxjs';
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { CustomerLinksConfig } from './dtos/customers';
import ErrorCodes from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import {
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
)
}
async getLinks(customerId: string) {
async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
try {
const result = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId }),
this.customerService.CustomerGetLinks({ customerId }),
);
return result.customer?.links || [];
return (result.links as CustomerLinksConfig) || null;
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
}
}
async setLinks(customerId: string, links: Link[]) {
async setLinks(customerId: string, links: CustomerLinksConfig) {
if (!customerId || !links) {
throw new HttpException(null, HttpStatus.BAD_REQUEST);
}
try {
return await firstValueFrom(
this.customerService.CustomerUpdate({
id: customerId,
links,
} as CustomerUpdateRequest),
this.customerService.CustomerSetLinks({
customerId,
links: links as CustomerSetLinksRequest['links'],
}),
);
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
+52 -9
View File
@@ -1,7 +1,6 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class CustomerLink implements Link {
export class CustomerLinkItem {
@ApiProperty()
href: string;
@ApiProperty()
@@ -9,15 +8,59 @@ export class CustomerLink implements Link {
@ApiProperty()
description: string;
@ApiPropertyOptional()
iconSrc: string;
iconSrc?: string;
}
export class CustomerSidebarLinkItem {
@ApiProperty()
type: 'link';
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiProperty()
link: string;
@ApiPropertyOptional()
icon?: string;
}
export class CustomerSidebarMenuItem {
@ApiProperty()
type: 'menu';
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiPropertyOptional()
icon?: string;
@ApiProperty({ type: [CustomerSidebarLinkItem] })
items: CustomerSidebarLinkItem[];
}
export class CustomerSidebarSection {
@ApiProperty({ type: Object })
title: Record<string, string>;
@ApiProperty({
type: 'array',
items: {
oneOf: [
{ $ref: '#/components/schemas/CustomerSidebarMenuItem' },
{ $ref: '#/components/schemas/CustomerSidebarLinkItem' },
],
},
})
items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
}
export class CustomerLinksConfig {
@ApiPropertyOptional({ type: [CustomerLinkItem] })
home?: CustomerLinkItem[];
@ApiPropertyOptional({ type: [CustomerSidebarSection] })
sidebar?: CustomerSidebarSection[];
}
export class CustomerLinkRequest {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
@ApiProperty({ type: CustomerLinksConfig })
links: CustomerLinksConfig;
}
export class CustomerLinksResponse {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
}
@ApiPropertyOptional({ type: CustomerLinksConfig })
links?: CustomerLinksConfig;
}
+5 -3
View File
@@ -11,17 +11,19 @@ export class TableColumns {
name: string;
@ApiProperty()
columns: string[];
@ApiProperty()
@ApiPropertyOptional({ type: [Column] })
references: Column[];
@ApiProperty()
identifier_columns: string[];
@ApiProperty()
destination: Record<'raw' | 'qualify', {
table_name: string;
table_schema: string;
}> | null;
@ApiProperty()
type: string;
@ApiPropertyOptional({ type: [String] })
identifier_columns?: string[];
@ApiPropertyOptional({ type: Column })
reference_column?: Column;
}
export class AvailableEntity {
@ApiProperty()
+1
View File
@@ -99,6 +99,7 @@ export class InputsController {
customer: info.customer,
});
this.logger.info(JSON.stringify(body))
const response = await this.inputService.create({ body, info });
return response;
+5
View File
@@ -164,6 +164,11 @@ export class InputsService {
const inputCreateGenericRequest: InputCreateGenericRequest = {
input: {
...body,
tables: (body.tables || []).map((table) => ({
...table,
identifier_columns: table.identifier_columns || [],
reference_column: table.reference_column || table.references?.[0],
})),
},
info,
};
+62 -95
View File
@@ -356,125 +356,92 @@ export class PipelinesService implements OnModuleInit {
info
)
const requests = [];
this.logger.info('Dynamo Response', updateInputResponse);
const jobsUpdated = [];
for (const [index, table] of updateInputDTO.tables.entries()) {
const id = `${pipelineIdFormat}_${index}`;
const jobUpdate = {
job_id: `${pipelineIdFormat}_${index}`,
}
if (table.memory) {
jobUpdate["memory"] = {
amount: table.memory * 1000
}
}
this.logger.info('Updating input reference for table', table.name);
const body = {}
let hasUpdateSyncMode = false;
const jobSyncMode = {}
if (table.columns) {
body['column_include_list'] = table.columns;
hasUpdateSyncMode = true;
jobSyncMode['column_include_list'] = table.columns;
}
if (table.reference_column) {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
hasUpdateSyncMode = true;
jobSyncMode['incremental_column_name'] = table.reference_column.name;
jobSyncMode['incremental_column_type'] = table.reference_column.type;
}
if (table.identifier_columns) {
body['primary_keys'] = table.identifier_columns;
}
this.logger.info('Request body', body);
const updateCollumns = this.platformAPI.proxy(
'PATCH',
`/jobs/${id}/input`,
user,
body
)
requests.push(updateCollumns);
if (table.memory) {
this.logger.info('Updating memory allocation for table', table.name);
const updateMemory = this.platformAPI.proxy(
'PUT',
`/jobs/${id}/memory`,
user,
{
amount: table.memory
}
)
requests.push(updateMemory);
hasUpdateSyncMode = true;
jobSyncMode['primary_keys'] = table.identifier_columns;
}
if (table.type) {
const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
requests.push(updateSyncMode);
hasUpdateSyncMode = true;
jobSyncMode['target_load_type'] = table.type;
}
}
this.logger.info('Create Platform Request for each JOB');
if(hasUpdateSyncMode) {
jobUpdate["sync_mode"] = jobSyncMode;
}
if (updateInputDTO.cron) {
const crnUpdatedRequest = new Promise(async (resolve, reject) => {
const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
if (response.error) {
this.logger.error('Error updating pipeline cron', response.error);
return reject(new ErrorBuilder(response.error));
if (Object.keys(table.destinations).length > 1) {
let hasChanges = false
const jobRenameTables = {
raw: {},
qualify: {}
}
this.logger.error('Pipeline cron updated successfully', response);
return resolve(response);
});
requests.push(crnUpdatedRequest);
if (Object.keys(table.destinations.raw).length > 1) {
hasChanges = true;
jobRenameTables.raw = table.destinations.raw;
}
if (Object.keys(table.destinations.qualify).length > 1) {
hasChanges = true;
jobRenameTables.qualify = table.destinations.qualify;
}
if (hasChanges) {
jobUpdate['rename_tables'] = jobRenameTables;
}
}
jobsUpdated.push(jobUpdate);
}
this.logger.info('Executing all request for the platform api');
this.logger.info('Request body:' + JSON.stringify({
jobs_updated: jobsUpdated
}));
const results = await Promise.allSettled(requests);
this.logger.info('Platform api response', results);
const response = await this.platformAPI.proxy(
'POST',
`/pipelines/${pipelineIdFormat}/batch-update`,
user,
{
job_updates: jobsUpdated
}
)
this.logger.info('Platform api response: ' + JSON.stringify(response));
return updateInputResponse;
}
private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) {
const body = {
target_load_type: table.type
}
if (table.type === 'incremental_with_qualify') {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
body['primary_keys'] = table.identifier_columns;
}
if (table.type === 'incremental') {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
}
this.logger.info('Updating pipeline sync mode', {
pipelineId,
body
});
return this.platformAPI.proxy(
"POST",
`/jobs/jdbc/${pipelineId}/sync-mode`,
user,
body
)
}
private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
try {
const response = await this.platformAPI.proxy(
'PATCH',
`/pipeline/${pipelineId}`,
user,
{
cron
}
);
return response
} catch (error) {
return {
error: error.message
}
}
}
}