mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-11 00:14:47 +00:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dcf7aed51c | ||
|
|
7fbce8b5be | ||
|
|
049eca9700 | ||
|
|
3bcd78bb32 | ||
|
|
f603b64679 | ||
|
|
b640465624 | ||
|
|
98282471a9 | ||
|
|
8983430889 | ||
|
|
d3aeca12cd | ||
|
|
ce619942a3 | ||
|
|
71b3f278a5 | ||
|
|
c92542ed90 | ||
|
|
69a9d78642 | ||
|
|
bf4f3cfd8c | ||
|
|
01afa69fcb | ||
|
|
80f3913fd2 | ||
|
|
10293ad6a9 |
@@ -4519,6 +4519,50 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/pipeline/{pipelineId}/pipeline_run/{runId}/cancel": {
|
||||
"post": {
|
||||
"operationId": "PlatformApiController_cancelPipelineRun",
|
||||
"summary": "Cancel a running pipeline run",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "pipelineId",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "runId",
|
||||
"required": true,
|
||||
"in": "path",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Platform API"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/platform/jobs/{jobId}/input": {
|
||||
"put": {
|
||||
"operationId": "PlatformApiController_updateJobInput",
|
||||
|
||||
Generated
+82
-9
@@ -17,7 +17,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "3.38.0-beta.31",
|
||||
"@dadosfera/protospack-v2": "3.40.0-beta.2",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -47,6 +47,7 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
@@ -1744,9 +1745,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.38.0-beta.31",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.31.tgz",
|
||||
"integrity": "sha512-IxnVEaVvQNumN/PQsnF+9CsFvmn6/0YyfXauoPQ1bccvqU8Lo+hKZv4qTBrR+urd05OWyg+dJBv/QtW45b04XQ==",
|
||||
"version": "3.40.0-beta.2",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz",
|
||||
"integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
@@ -2928,6 +2929,20 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
|
||||
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
|
||||
"engines": [
|
||||
"node >= 0.8"
|
||||
],
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^2.2.2",
|
||||
"typedarray": "^0.0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
|
||||
"version": "0.5.4",
|
||||
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
|
||||
@@ -3097,6 +3112,24 @@
|
||||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/multer": {
|
||||
"version": "1.4.4-lts.1",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
|
||||
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
|
||||
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.0.0",
|
||||
"concat-stream": "^1.5.2",
|
||||
"mkdirp": "^0.5.4",
|
||||
"object-assign": "^4.1.1",
|
||||
"type-is": "^1.6.4",
|
||||
"xtend": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
|
||||
@@ -3121,6 +3154,25 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
|
||||
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
|
||||
"dependencies": {
|
||||
"core-util-is": "~1.0.0",
|
||||
"inherits": "~2.0.3",
|
||||
"isarray": "~1.0.0",
|
||||
"process-nextick-args": "~2.0.0",
|
||||
"safe-buffer": "~5.1.1",
|
||||
"string_decoder": "~1.1.1",
|
||||
"util-deprecate": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
@@ -3195,6 +3247,19 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
|
||||
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
|
||||
"dependencies": {
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
|
||||
},
|
||||
"node_modules/@nestjs/platform-express/node_modules/tslib": {
|
||||
"version": "2.5.3",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
|
||||
@@ -5534,9 +5599,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "0.30.3",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.3.tgz",
|
||||
"integrity": "sha512-5/tmEb6TmE/ax3mdXBc/Mi6YdPGxQsv+0p5YlciXWt3PHIn0VamqCXhRMtScnwY3lbgSXLneOuXAKUhgmSRpwg==",
|
||||
"version": "0.30.2",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.2.tgz",
|
||||
"integrity": "sha512-0pE4RQ4UQi1jKY6p7u6i1Tkzqmu+d+/tHS7Q7rKunWLB9WyilBTpHHpXzPNMDj5hTbK0B0PTLSz07yqMBiF6xg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.4",
|
||||
@@ -6509,7 +6574,6 @@
|
||||
"engines": [
|
||||
"node >= 6.0"
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
@@ -9128,6 +9192,11 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/isarray": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
|
||||
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
|
||||
},
|
||||
"node_modules/isexe": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
|
||||
@@ -10725,7 +10794,6 @@
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
|
||||
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"append-field": "^1.0.0",
|
||||
"busboy": "^1.6.0",
|
||||
@@ -11728,6 +11796,11 @@
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/process-nextick-args": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
|
||||
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
|
||||
},
|
||||
"node_modules/process-warning": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
|
||||
|
||||
+3
-3
@@ -10,7 +10,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact",
|
||||
"prebuild": "rimraf dist",
|
||||
"build": "nest build",
|
||||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
||||
@@ -35,7 +35,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "3.38.0-beta.31",
|
||||
"@dadosfera/protospack-v2": "3.40.0-beta.2",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -65,6 +65,7 @@
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"multer": "^2.0.2",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
@@ -80,7 +81,6 @@
|
||||
"swagger-ui-express": "^4.6.3"
|
||||
},
|
||||
"overrides": {
|
||||
"multer": "2.0.2",
|
||||
"form-data": "^4.0.4",
|
||||
"body-parser": "^1.20.3",
|
||||
"cross-spawn": "^7.0.5",
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
InputCreateS3Request,
|
||||
InputNewCreateRequest,
|
||||
InputUpdateResponse,
|
||||
RollbackInputRequest,
|
||||
TestConnectionRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
@@ -209,7 +210,8 @@ export class InputsService {
|
||||
// this.validateCron({ ...data, info });
|
||||
try {
|
||||
const {
|
||||
nimbusUpdates,
|
||||
tablesUpdate,
|
||||
dataAssetUpdate,
|
||||
input
|
||||
} = await this.OLD_inputClient.update({
|
||||
id,
|
||||
@@ -222,13 +224,29 @@ export class InputsService {
|
||||
);
|
||||
return {
|
||||
input: updateInputResponse,
|
||||
nimbusUpdates
|
||||
tablesUpdate,
|
||||
dataAssetUpdate
|
||||
};
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
async rollbackUpdate(
|
||||
data: RollbackInputRequest
|
||||
) {
|
||||
this.logger.info('PipelinesClientService - rollbackUpdate');
|
||||
this.logger.info('Rolling back input update with data: ' + JSON.stringify(data));
|
||||
const updatePipelineResponse = await lastValueFrom(
|
||||
this.inputWriteService.RollbackInputUpdate(
|
||||
data
|
||||
),
|
||||
);
|
||||
this.logger.info('Done');
|
||||
|
||||
return updatePipelineResponse;
|
||||
}
|
||||
|
||||
async remove(idRequest: IIdRequest) {
|
||||
return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
|
||||
}
|
||||
|
||||
@@ -28,11 +28,13 @@ import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
import { PlatformApiService } from '../platform-api/platform-api.service';
|
||||
import { Info, NewTable } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { NimbusUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import axios from 'axios';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { AxiosError } from 'axios';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
|
||||
type RollbackPromise = () => Promise<any>;
|
||||
|
||||
export class PipelinesService implements OnModuleInit {
|
||||
logger: DadosferaLogger;
|
||||
pipelineWriteService: WriteService.PipelineV2WriteService;
|
||||
@@ -352,33 +354,116 @@ export class PipelinesService implements OnModuleInit {
|
||||
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
|
||||
this.logger.info('InputClientService - Update');
|
||||
|
||||
this.logger.info('Update Dynamo Reference');
|
||||
const {
|
||||
input: oldInput
|
||||
} = await this.inputsService.findOne({
|
||||
id: inputId,
|
||||
info: info
|
||||
});
|
||||
|
||||
this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
|
||||
const pipelineIdFormat = pipelineId.split('-').join('_');
|
||||
console.log("updatePipelineInput", info, updateInputDTO)
|
||||
const rollback: RollbackPromise[] = [];
|
||||
|
||||
const updateInputResponse = await this.inputsService.update(
|
||||
inputId,
|
||||
updateInputDTO,
|
||||
info
|
||||
)
|
||||
|
||||
this.logger.info(JSON.stringify(updateInputResponse))
|
||||
await this.updateNimbus(info.customer, updateInputResponse.nimbusUpdates);
|
||||
|
||||
await this.updatePlatformJobs(
|
||||
pipelineIdFormat,
|
||||
updateInputDTO,
|
||||
user
|
||||
);
|
||||
|
||||
const inputRollback = () => {
|
||||
this.logger.info("exec rollback to input: " + JSON.stringify(oldInput));
|
||||
return this.inputsService.rollbackUpdate(
|
||||
{
|
||||
id: inputId,
|
||||
dataAssetUpdate: updateInputResponse.dataAssetUpdate,
|
||||
tables: oldInput.tables,
|
||||
info
|
||||
}
|
||||
) as Promise<any>;
|
||||
}
|
||||
|
||||
rollback.push(inputRollback);
|
||||
|
||||
this.logger.info("Input Update Response: " + JSON.stringify(updateInputResponse))
|
||||
|
||||
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
|
||||
|
||||
nimbusUpdates.forEach(update => {
|
||||
const nimbusRollback = () => {
|
||||
return this.nimbusService.renameTable(
|
||||
info.customer,
|
||||
update.database,
|
||||
{
|
||||
table_name: update.table_name,
|
||||
table_schema: update.table_schema
|
||||
},
|
||||
{
|
||||
table_name: update.old_table_name,
|
||||
table_schema: update.old_table_schema
|
||||
}
|
||||
);
|
||||
}
|
||||
rollback.push(nimbusRollback);
|
||||
});
|
||||
|
||||
try {
|
||||
await this.updateNimbus(info.customer, nimbusUpdates);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
if (error instanceof AxiosError) {
|
||||
this.logger.error(JSON.stringify(error.response.data));
|
||||
}
|
||||
await this.executeRenameRollback(rollback);
|
||||
|
||||
throw new Error("Error Nimbus updating tables");
|
||||
}
|
||||
|
||||
try {
|
||||
await this.updatePlatformJobs(
|
||||
pipelineIdFormat,
|
||||
updateInputDTO,
|
||||
user
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
await this.executeRenameRollback(rollback)
|
||||
throw new Error("Error Platform API updating jobs");
|
||||
}
|
||||
|
||||
return updateInputResponse;
|
||||
}
|
||||
|
||||
private async updateNimbus(customer: string, changes: NimbusUpdate[]) {
|
||||
private async executeRenameRollback(request: RollbackPromise[]) {
|
||||
this.logger.info('rollback steps: ' + request.length)
|
||||
const result = await Promise.allSettled(request.map(func => func()));
|
||||
result.forEach(promise => {
|
||||
this.logger.info("Promise finish with status: " + promise.status)
|
||||
|
||||
if (promise.status === "rejected") {
|
||||
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}))
|
||||
}
|
||||
|
||||
if (promise.status === "fulfilled") {
|
||||
this.logger.info("success with: " + JSON.stringify(promise.value || {}))
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
private async updateNimbus(customer: string, changes: TableUpdate[]) {
|
||||
// throw new Error("teste error nimbus");
|
||||
this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
|
||||
if(!changes || changes.length === 0) return;
|
||||
|
||||
const requests = changes.map(change => {
|
||||
return this.nimbusService.renameTable(customer, change.database, change.old, change.new);
|
||||
return this.nimbusService.renameTable(customer, change.database, {
|
||||
table_name: change.old_table_name,
|
||||
table_schema: change.old_table_schema
|
||||
}, {
|
||||
table_name: change.table_name,
|
||||
table_schema: change.table_schema
|
||||
});
|
||||
})
|
||||
|
||||
const values = await Promise.allSettled(requests);
|
||||
@@ -386,6 +471,20 @@ export class PipelinesService implements OnModuleInit {
|
||||
const success = values.map(request => request.status === "fulfilled")
|
||||
|
||||
this.logger.info("Updates with succes: " + success.length);
|
||||
|
||||
values.forEach(promise => {
|
||||
this.logger.info("Promise finish with status: " + promise.status)
|
||||
|
||||
if (promise.status === "rejected") {
|
||||
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}));
|
||||
throw new Error(promise.reason );
|
||||
}
|
||||
|
||||
if (promise.status === "fulfilled") {
|
||||
this.logger.info("success with: " + JSON.stringify(promise.value || {}));
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
|
||||
@@ -396,6 +495,10 @@ export class PipelinesService implements OnModuleInit {
|
||||
job_id: `${pipelineId}_${index}`,
|
||||
}
|
||||
|
||||
if (table.type !== "incremental_with_qualify") {
|
||||
delete table.destinations?.qualify;
|
||||
}
|
||||
|
||||
if (table.memory) {
|
||||
jobUpdate["memory"] = {
|
||||
amount: table.memory * 1000
|
||||
|
||||
@@ -845,6 +845,24 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Post('pipeline/:pipelineId/pipeline_run/:runId/cancel')
|
||||
@ApiOperation({ summary: 'Cancel a running pipeline run' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async cancelPipelineRun(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('runId') runId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const normalizedRunId = this.normalizePipelineId(runId);
|
||||
|
||||
return this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== JOBS - COLUMN EDITING ROUTES ====================
|
||||
|
||||
@Put('jobs/:jobId/input')
|
||||
|
||||
@@ -34,7 +34,7 @@ export class NimbusService {
|
||||
const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`;
|
||||
|
||||
try {
|
||||
this.logger.info("Request for PATCH" + nimbusUrl + path);
|
||||
this.logger.info("Request for PATCH " + nimbusUrl + path);
|
||||
this.logger.info("Payload: " + JSON.stringify(update));
|
||||
const { data } = await axios.patch(nimbusUrl + path, {
|
||||
table_name: update.table_name,
|
||||
|
||||
Reference in New Issue
Block a user