Compare commits

...
Author SHA1 Message Date
marcos-silva-rodrigues dcf7aed51c MERGE: resolve package conflicts 2026-03-30 12:05:36 -03:00
marcos-silva-rodrigues 7fbce8b5be FEAT: update input 2026-03-30 12:02:27 -03:00
vinicius gadea 049eca9700 Merge pull request #472 from dadosfera/feat/cancel-pipeline-run-beta
FEAT: update protospack-v2 to version 3.40.0-beta.1 and adjust relate…
2026-03-26 14:54:32 -03:00
viniciusgadea 3bcd78bb32 FEAT: update protospack-v2 to version 3.40.0-beta.1 and adjust related scripts 2026-03-26 14:53:33 -03:00
vinicius gadea f603b64679 Merge pull request #471 from dadosfera/feat/cancel-pipeline-run-beta
Feat/cancel pipeline run beta
2026-03-26 11:29:09 -03:00
viniciusgadea b640465624 Merge remote-tracking branch 'origin/beta' into feat/cancel-pipeline-run-beta 2026-03-26 11:19:30 -03:00
viniciusgadea 98282471a9 FIX: remove unused data asset methods from ElasticsearchService 2026-03-26 11:08:40 -03:00
viniciusgadea 8983430889 REF: remove last_run_status tracking and related Elasticsearch update logic from pipeline cancellation 2026-03-26 10:32:09 -03:00
viniciusgadea d3aeca12cd FIX: remove last_run_canceled_at from PipelineDocument and update last_run_status handling 2026-03-26 10:32:09 -03:00
viniciusgadea ce619942a3 FEAT: add endpoint to cancel running pipeline runs and update Elasticsearch status 2026-03-26 10:32:09 -03:00
Marcos Rodrigues Silva 71b3f278a5 Merge pull request #469 from dadosfera/feat/override-menu2
Feat/override menu2
2026-03-25 18:02:14 -03:00
viniciusgadea c92542ed90 FEAT: update protospack-v2 version to 3.39.0 in package.json and package-lock.json 2026-03-25 15:53:02 -03:00
viniciusgadeaandClaude Sonnet 4.6 69a9d78642 FIX: add multer@2.0.2 and update package-lock.json — missing transitive dep of protospack-v2@3.38.0-beta.30
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-25 15:41:02 -03:00
viniciusgadea bf4f3cfd8c FEAT: update CustomerSidebarSection and related DTOs to use object type for title; update protospack-v2 version; update from id to customerId 2026-03-25 15:40:57 -03:00
viniciusgadeaandClaude Sonnet 4.6 01afa69fcb FIX: remove duplicate identifier_columns declaration in TableColumns DTO
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-25 15:37:15 -03:00
viniciusgadea 80f3913fd2 FEAT: update CustomerSidebarSection to support both menu and link items 2026-03-25 15:24:59 -03:00
viniciusgadea 10293ad6a9 FEAT: update customer links handling and DTOs for improved structure sidebar. update protospack-v2 2026-03-25 15:24:59 -03:00
7 changed files with 287 additions and 31 deletions
+44
View File
@@ -4519,6 +4519,50 @@
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run/{runId}/cancel": {
"post": {
"operationId": "PlatformApiController_cancelPipelineRun",
"summary": "Cancel a running pipeline run",
"parameters": [
{
"name": "pipelineId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "runId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/{jobId}/input": {
"put": {
"operationId": "PlatformApiController_updateJobInput",
+82 -9
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.31",
"@dadosfera/protospack-v2": "3.40.0-beta.2",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -47,6 +47,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
@@ -1744,9 +1745,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.31",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.31.tgz",
"integrity": "sha512-IxnVEaVvQNumN/PQsnF+9CsFvmn6/0YyfXauoPQ1bccvqU8Lo+hKZv4qTBrR+urd05OWyg+dJBv/QtW45b04XQ==",
"version": "3.40.0-beta.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz",
"integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
@@ -2928,6 +2929,20 @@
"node": ">= 0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"engines": [
"node >= 0.8"
],
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
}
},
"node_modules/@nestjs/platform-express/node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -3097,6 +3112,24 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/@nestjs/platform-express/node_modules/multer": {
"version": "1.4.4-lts.1",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
"integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
"deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.0.0",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.4",
"object-assign": "^4.1.1",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/negotiator": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3121,6 +3154,25 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
"integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
"dependencies": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3195,6 +3247,19 @@
"node": ">= 0.8"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"dependencies": {
"safe-buffer": "~5.1.0"
}
},
"node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"node_modules/@nestjs/platform-express/node_modules/tslib": {
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
@@ -5534,9 +5599,9 @@
}
},
"node_modules/axios": {
"version": "0.30.3",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.3.tgz",
"integrity": "sha512-5/tmEb6TmE/ax3mdXBc/Mi6YdPGxQsv+0p5YlciXWt3PHIn0VamqCXhRMtScnwY3lbgSXLneOuXAKUhgmSRpwg==",
"version": "0.30.2",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.2.tgz",
"integrity": "sha512-0pE4RQ4UQi1jKY6p7u6i1Tkzqmu+d+/tHS7Q7rKunWLB9WyilBTpHHpXzPNMDj5hTbK0B0PTLSz07yqMBiF6xg==",
"license": "MIT",
"dependencies": {
"follow-redirects": "^1.15.4",
@@ -6509,7 +6574,6 @@
"engines": [
"node >= 6.0"
],
"license": "MIT",
"dependencies": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
@@ -9128,6 +9192,11 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
},
"node_modules/isexe": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -10725,7 +10794,6 @@
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
"integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
"license": "MIT",
"dependencies": {
"append-field": "^1.0.0",
"busboy": "^1.6.0",
@@ -11728,6 +11796,11 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"node_modules/process-warning": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",
+3 -3
View File
@@ -10,7 +10,7 @@
},
"scripts": {
"co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
"proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact",
"prebuild": "rimraf dist",
"build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
@@ -35,7 +35,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.31",
"@dadosfera/protospack-v2": "3.40.0-beta.2",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -65,6 +65,7 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"multer": "^2.0.2",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
@@ -80,7 +81,6 @@
"swagger-ui-express": "^4.6.3"
},
"overrides": {
"multer": "2.0.2",
"form-data": "^4.0.4",
"body-parser": "^1.20.3",
"cross-spawn": "^7.0.5",
+20 -2
View File
@@ -18,6 +18,7 @@ import {
InputCreateS3Request,
InputNewCreateRequest,
InputUpdateResponse,
RollbackInputRequest,
TestConnectionRequest,
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -209,7 +210,8 @@ export class InputsService {
// this.validateCron({ ...data, info });
try {
const {
nimbusUpdates,
tablesUpdate,
dataAssetUpdate,
input
} = await this.OLD_inputClient.update({
id,
@@ -222,13 +224,29 @@ export class InputsService {
);
return {
input: updateInputResponse,
nimbusUpdates
tablesUpdate,
dataAssetUpdate
};
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async rollbackUpdate(
data: RollbackInputRequest
) {
this.logger.info('PipelinesClientService - rollbackUpdate');
this.logger.info('Rolling back input update with data: ' + JSON.stringify(data));
const updatePipelineResponse = await lastValueFrom(
this.inputWriteService.RollbackInputUpdate(
data
),
);
this.logger.info('Done');
return updatePipelineResponse;
}
async remove(idRequest: IIdRequest) {
return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
}
+119 -16
View File
@@ -28,11 +28,13 @@ import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import { PlatformApiService } from '../platform-api/platform-api.service';
import { Info, NewTable } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { NimbusUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import axios from 'axios';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { AxiosError } from 'axios';
import { NimbusService } from 'src/services/nimbus/nimbus.service';
type RollbackPromise = () => Promise<any>;
export class PipelinesService implements OnModuleInit {
logger: DadosferaLogger;
pipelineWriteService: WriteService.PipelineV2WriteService;
@@ -352,33 +354,116 @@ export class PipelinesService implements OnModuleInit {
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
this.logger.info('InputClientService - Update');
this.logger.info('Update Dynamo Reference');
const {
input: oldInput
} = await this.inputsService.findOne({
id: inputId,
info: info
});
this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
const pipelineIdFormat = pipelineId.split('-').join('_');
console.log("updatePipelineInput", info, updateInputDTO)
const rollback: RollbackPromise[] = [];
const updateInputResponse = await this.inputsService.update(
inputId,
updateInputDTO,
info
)
this.logger.info(JSON.stringify(updateInputResponse))
await this.updateNimbus(info.customer, updateInputResponse.nimbusUpdates);
await this.updatePlatformJobs(
pipelineIdFormat,
updateInputDTO,
user
);
const inputRollback = () => {
this.logger.info("exec rollback to input: " + JSON.stringify(oldInput));
return this.inputsService.rollbackUpdate(
{
id: inputId,
dataAssetUpdate: updateInputResponse.dataAssetUpdate,
tables: oldInput.tables,
info
}
) as Promise<any>;
}
rollback.push(inputRollback);
this.logger.info("Input Update Response: " + JSON.stringify(updateInputResponse))
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
nimbusUpdates.forEach(update => {
const nimbusRollback = () => {
return this.nimbusService.renameTable(
info.customer,
update.database,
{
table_name: update.table_name,
table_schema: update.table_schema
},
{
table_name: update.old_table_name,
table_schema: update.old_table_schema
}
);
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
}
try {
await this.updatePlatformJobs(
pipelineIdFormat,
updateInputDTO,
user
);
} catch (error) {
this.logger.error(error);
await this.executeRenameRollback(rollback)
throw new Error("Error Platform API updating jobs");
}
return updateInputResponse;
}
private async updateNimbus(customer: string, changes: NimbusUpdate[]) {
private async executeRenameRollback(request: RollbackPromise[]) {
this.logger.info('rollback steps: ' + request.length)
const result = await Promise.allSettled(request.map(func => func()));
result.forEach(promise => {
this.logger.info("Promise finish with status: " + promise.status)
if (promise.status === "rejected") {
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}))
}
if (promise.status === "fulfilled") {
this.logger.info("success with: " + JSON.stringify(promise.value || {}))
}
});
}
private async updateNimbus(customer: string, changes: TableUpdate[]) {
// throw new Error("teste error nimbus");
this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
if(!changes || changes.length === 0) return;
const requests = changes.map(change => {
return this.nimbusService.renameTable(customer, change.database, change.old, change.new);
return this.nimbusService.renameTable(customer, change.database, {
table_name: change.old_table_name,
table_schema: change.old_table_schema
}, {
table_name: change.table_name,
table_schema: change.table_schema
});
})
const values = await Promise.allSettled(requests);
@@ -386,6 +471,20 @@ export class PipelinesService implements OnModuleInit {
const success = values.map(request => request.status === "fulfilled")
this.logger.info("Updates with succes: " + success.length);
values.forEach(promise => {
this.logger.info("Promise finish with status: " + promise.status)
if (promise.status === "rejected") {
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}));
throw new Error(promise.reason );
}
if (promise.status === "fulfilled") {
this.logger.info("success with: " + JSON.stringify(promise.value || {}));
}
});
}
async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
@@ -396,6 +495,10 @@ export class PipelinesService implements OnModuleInit {
job_id: `${pipelineId}_${index}`,
}
if (table.type !== "incremental_with_qualify") {
delete table.destinations?.qualify;
}
if (table.memory) {
jobUpdate["memory"] = {
amount: table.memory * 1000
@@ -845,6 +845,24 @@ export class PlatformApiController {
);
}
@Post('pipeline/:pipelineId/pipeline_run/:runId/cancel')
@ApiOperation({ summary: 'Cancel a running pipeline run' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async cancelPipelineRun(
@Param('pipelineId') pipelineId: string,
@Param('runId') runId: string,
@User() user: RequestUser,
) {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const normalizedRunId = this.normalizePipelineId(runId);
return this.platformApiService.proxy(
'POST',
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
user,
);
}
// ==================== JOBS - COLUMN EDITING ROUTES ====================
@Put('jobs/:jobId/input')
+1 -1
View File
@@ -34,7 +34,7 @@ export class NimbusService {
const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`;
try {
this.logger.info("Request for PATCH" + nimbusUrl + path);
this.logger.info("Request for PATCH " + nimbusUrl + path);
this.logger.info("Payload: " + JSON.stringify(update));
const { data } = await axios.patch(nimbusUrl + path, {
table_name: update.table_name,