mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-04 20:59:08 +00:00
Compare commits
30
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
92ba6cf454 | ||
|
|
1664ac06ff | ||
|
|
271174176b | ||
|
|
dcf7aed51c | ||
|
|
7fbce8b5be | ||
|
|
049eca9700 | ||
|
|
3bcd78bb32 | ||
|
|
f603b64679 | ||
|
|
b640465624 | ||
|
|
98282471a9 | ||
|
|
8983430889 | ||
|
|
d3aeca12cd | ||
|
|
ce619942a3 | ||
|
|
71b3f278a5 | ||
|
|
c92542ed90 | ||
|
|
69a9d78642 | ||
|
|
bf4f3cfd8c | ||
|
|
01afa69fcb | ||
|
|
80f3913fd2 | ||
|
|
10293ad6a9 | ||
|
|
e4d0c9c3e6 | ||
|
|
21c57e5620 | ||
|
|
959210e354 | ||
|
|
f10953e949 | ||
|
|
13903b9bb9 | ||
|
|
2e3a13d421 | ||
|
|
fa17fc3001 | ||
|
|
254a638392 | ||
|
|
9a29ef5401 | ||
|
|
3f21faaa66 |
+1100
-124
File diff suppressed because it is too large
Load Diff
Generated
+5
-4
@@ -17,7 +17,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
|
||||
"@dadosfera/protospack-v2": "3.40.0-beta.2",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -1745,9 +1745,10 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@dadosfera/protospack-v2": {
|
||||
"version": "3.38.0-beta.30",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.30.tgz",
|
||||
"integrity": "sha512-EePtEV4Bjr47BCtQgeZP8WzappDv1mRWMro15ms+Vdev/FIVBydMew/BtK3spzI7KHWJniB0Ctsvpcdj2GiTDA==",
|
||||
"version": "3.40.0-beta.2",
|
||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz",
|
||||
"integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"rxjs": "^7.5.5"
|
||||
|
||||
+2
-3
@@ -10,7 +10,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
|
||||
"proto-update": "npm i @dadosfera/protospack-v2@v3.40.0-beta.1 --save-exact",
|
||||
"prebuild": "rimraf dist",
|
||||
"build": "nest build",
|
||||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
|
||||
@@ -35,7 +35,7 @@
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "^3.38.0-beta.30",
|
||||
"@dadosfera/protospack-v2": "^3.40.0-beta.3",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -81,7 +81,6 @@
|
||||
"swagger-ui-express": "^4.6.3"
|
||||
},
|
||||
"overrides": {
|
||||
"multer": "2.0.2",
|
||||
"form-data": "^4.0.4",
|
||||
"body-parser": "^1.20.3",
|
||||
"cross-spawn": "^7.0.5",
|
||||
|
||||
@@ -489,6 +489,7 @@ export class AuthController {
|
||||
const userDto = {
|
||||
id: api_key.user_id,
|
||||
name: api_key.username,
|
||||
email: api_key.username,
|
||||
customer: {
|
||||
id: api_key.customer_id,
|
||||
name: api_key.customer_name,
|
||||
|
||||
@@ -437,7 +437,8 @@ export class AuthClientService implements OnModuleInit {
|
||||
|
||||
const userDto: UserDTO = {
|
||||
id: user.id,
|
||||
name: user.username,
|
||||
name: user.name,
|
||||
email: user.email,
|
||||
jobTitle: user?.jobTitle || null,
|
||||
department: user?.department || null,
|
||||
hierarchy: user?.hierarchy || null,
|
||||
|
||||
@@ -144,6 +144,7 @@ export interface BulkEditResponse {
|
||||
export type UserDTO = {
|
||||
id: string,
|
||||
name: string,
|
||||
email: string,
|
||||
jobTitle?: string,
|
||||
department?: string,
|
||||
hierarchy?: string,
|
||||
|
||||
@@ -99,6 +99,7 @@ export class InputsController {
|
||||
customer: info.customer,
|
||||
});
|
||||
|
||||
this.logger.info(JSON.stringify(body))
|
||||
const response = await this.inputService.create({ body, info });
|
||||
|
||||
return response;
|
||||
|
||||
@@ -17,6 +17,8 @@ import {
|
||||
InputCreateGenericRequest,
|
||||
InputCreateS3Request,
|
||||
InputNewCreateRequest,
|
||||
InputUpdateResponse,
|
||||
RollbackInputRequest,
|
||||
TestConnectionRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
@@ -71,8 +73,8 @@ export class InputsService {
|
||||
objectCamelToSnake(createInputResponse);
|
||||
return createInputResponse;
|
||||
},
|
||||
update: async (updateInputDTO: UpdateInputRequest) => {
|
||||
this.logger.info('InputClientService - Update');
|
||||
update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
|
||||
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
|
||||
const updateInputResponse = await lastValueFrom(
|
||||
this.inputWriteService.InputUpdate(updateInputDTO),
|
||||
);
|
||||
@@ -207,21 +209,44 @@ export class InputsService {
|
||||
async update(id: string, data, info: Info) {
|
||||
// this.validateCron({ ...data, info });
|
||||
try {
|
||||
const updateInputResponse: any = await this.OLD_inputClient.update({
|
||||
const {
|
||||
tablesUpdate,
|
||||
dataAssetUpdate,
|
||||
input
|
||||
} = await this.OLD_inputClient.update({
|
||||
id,
|
||||
info,
|
||||
...data,
|
||||
info,
|
||||
});
|
||||
|
||||
updateInputResponse.input = this.adjustInputPayload(
|
||||
updateInputResponse?.input,
|
||||
const updateInputResponse = this.adjustInputPayload(
|
||||
input,
|
||||
);
|
||||
return updateInputResponse;
|
||||
return {
|
||||
input: updateInputResponse,
|
||||
tablesUpdate,
|
||||
dataAssetUpdate
|
||||
};
|
||||
} catch (err) {
|
||||
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||
}
|
||||
}
|
||||
|
||||
async rollbackUpdate(
|
||||
data: RollbackInputRequest
|
||||
) {
|
||||
this.logger.info('PipelinesClientService - rollbackUpdate');
|
||||
this.logger.info('Rolling back input update with data: ' + JSON.stringify(data));
|
||||
const updatePipelineResponse = await lastValueFrom(
|
||||
this.inputWriteService.RollbackInputUpdate(
|
||||
data
|
||||
),
|
||||
);
|
||||
this.logger.info('Done');
|
||||
|
||||
return updatePipelineResponse;
|
||||
}
|
||||
|
||||
async remove(idRequest: IIdRequest) {
|
||||
return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
|
||||
}
|
||||
|
||||
@@ -224,11 +224,12 @@ export class PipelinesController {
|
||||
|
||||
const result = await this.pipelinesClientService
|
||||
.findOne({ id }, metadata)
|
||||
.then((res) => {
|
||||
.then(async (res) => {
|
||||
//{pipeline:{tables: {tables: [], input_id: ''}}}
|
||||
let tables = JSON.parse(res.pipeline.config.tables);
|
||||
const input_id = tables?.input_id;
|
||||
if (tables?.tables) tables = tables.tables;
|
||||
const parsed = JSON.parse(res.pipeline.config.tables);
|
||||
const input_id = parsed?.input_id;
|
||||
const tables = parsed?.tables ?? parsed;
|
||||
|
||||
Object.assign(res.pipeline, {
|
||||
transformations: res.pipeline.transformations
|
||||
? JSON.parse(res.pipeline.transformations)
|
||||
@@ -294,13 +295,14 @@ export class PipelinesController {
|
||||
) {
|
||||
this.logger.info('PipelinesController - update', { user });
|
||||
|
||||
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const info: Info = {
|
||||
user_id: user.user_id,
|
||||
customer: user.customer_name,
|
||||
customer_id: user.customer_id,
|
||||
pipeline_id: pipelineId
|
||||
};
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
|
||||
@@ -12,6 +12,8 @@ import { ConnectorModule } from '../connector/connector.module';
|
||||
import { InputsModule } from '../inputs/inputs.module';
|
||||
import { TransformationsModule } from '../transformations/transformations.module';
|
||||
import { PlatformApiModule } from '../platform-api/platform-api.module';
|
||||
import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
|
||||
const client = new PipelinesClientConfiguration();
|
||||
|
||||
@@ -22,10 +24,11 @@ const client = new PipelinesClientConfiguration();
|
||||
ConnectorModule,
|
||||
InputsModule,
|
||||
TransformationsModule,
|
||||
PlatformApiModule
|
||||
PlatformApiModule,
|
||||
NimbusServicesModule
|
||||
],
|
||||
controllers: [PipelinesController],
|
||||
providers: [PipelinesService, DadosferaLogger],
|
||||
providers: [PipelinesService, DadosferaLogger, NimbusService],
|
||||
exports: [PipelinesService],
|
||||
})
|
||||
export class PipelinesV2Module {}
|
||||
|
||||
@@ -29,6 +29,11 @@ import ErrorCodes from 'src/utils/errorCodes';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
import { PlatformApiService } from '../platform-api/platform-api.service';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||
import { AxiosError } from 'axios';
|
||||
import { NimbusService } from 'src/services/nimbus/nimbus.service';
|
||||
|
||||
type RollbackPromise = () => Promise<any>;
|
||||
|
||||
export class PipelinesService implements OnModuleInit {
|
||||
logger: DadosferaLogger;
|
||||
@@ -42,7 +47,8 @@ export class PipelinesService implements OnModuleInit {
|
||||
private readonly connectorService: ConnectorClientService,
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly transformationsService: TransformationsService,
|
||||
private readonly platformAPI: PlatformApiService
|
||||
private readonly platformAPI: PlatformApiService,
|
||||
private readonly nimbusService: NimbusService
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
@@ -348,133 +354,226 @@ export class PipelinesService implements OnModuleInit {
|
||||
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
|
||||
this.logger.info('InputClientService - Update');
|
||||
|
||||
this.logger.info('Update Dynamo Reference');
|
||||
const {
|
||||
input: oldInput
|
||||
} = await this.inputsService.findOne({
|
||||
id: inputId,
|
||||
info: info
|
||||
});
|
||||
|
||||
this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
|
||||
const pipelineIdFormat = pipelineId.split('-').join('_');
|
||||
const rollback: RollbackPromise[] = [];
|
||||
|
||||
const updateInputResponse = await this.inputsService.update(
|
||||
inputId,
|
||||
updateInputDTO,
|
||||
info
|
||||
)
|
||||
);
|
||||
|
||||
const requests = [];
|
||||
const inputRollback = () => {
|
||||
this.logger.info("exec rollback to input: " + JSON.stringify(oldInput));
|
||||
return this.inputsService.rollbackUpdate(
|
||||
{
|
||||
id: inputId,
|
||||
dataAssetUpdate: updateInputResponse.dataAssetUpdate,
|
||||
tables: oldInput.tables,
|
||||
info
|
||||
}
|
||||
) as Promise<any>;
|
||||
}
|
||||
|
||||
this.logger.info('Dynamo Response', updateInputResponse);
|
||||
rollback.push(inputRollback);
|
||||
|
||||
this.logger.info("Input Update Response: " + JSON.stringify(updateInputResponse))
|
||||
|
||||
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
|
||||
|
||||
nimbusUpdates.forEach(update => {
|
||||
const nimbusRollback = () => {
|
||||
return this.nimbusService.renameTable(
|
||||
info.customer,
|
||||
update.database,
|
||||
{
|
||||
table_name: update.table_name,
|
||||
table_schema: update.table_schema
|
||||
},
|
||||
{
|
||||
table_name: update.old_table_name,
|
||||
table_schema: update.old_table_schema
|
||||
}
|
||||
);
|
||||
}
|
||||
rollback.push(nimbusRollback);
|
||||
});
|
||||
|
||||
try {
|
||||
await this.updateNimbus(info.customer, nimbusUpdates);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
if (error instanceof AxiosError) {
|
||||
this.logger.error(JSON.stringify(error.response.data));
|
||||
}
|
||||
await this.executeRenameRollback(rollback);
|
||||
|
||||
throw new Error("Error Nimbus updating tables");
|
||||
}
|
||||
|
||||
try {
|
||||
await this.updatePlatformJobs(
|
||||
pipelineIdFormat,
|
||||
updateInputDTO,
|
||||
user
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
await this.executeRenameRollback(rollback)
|
||||
throw new Error("Error Platform API updating jobs");
|
||||
}
|
||||
|
||||
return updateInputResponse;
|
||||
}
|
||||
|
||||
private async executeRenameRollback(request: RollbackPromise[]) {
|
||||
this.logger.info('rollback steps: ' + request.length)
|
||||
const result = await Promise.allSettled(request.map(func => func()));
|
||||
result.forEach(promise => {
|
||||
this.logger.info("Promise finish with status: " + promise.status)
|
||||
|
||||
if (promise.status === "rejected") {
|
||||
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}))
|
||||
}
|
||||
|
||||
if (promise.status === "fulfilled") {
|
||||
this.logger.info("success with: " + JSON.stringify(promise.value || {}))
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
private async updateNimbus(customer: string, changes: TableUpdate[]) {
|
||||
// throw new Error("teste error nimbus");
|
||||
this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
|
||||
if(!changes || changes.length === 0) return;
|
||||
|
||||
const requests = changes.map(change => {
|
||||
return this.nimbusService.renameTable(customer, change.database, {
|
||||
table_name: change.old_table_name,
|
||||
table_schema: change.old_table_schema
|
||||
}, {
|
||||
table_name: change.table_name,
|
||||
table_schema: change.table_schema
|
||||
});
|
||||
})
|
||||
|
||||
const values = await Promise.allSettled(requests);
|
||||
|
||||
const success = values.map(request => request.status === "fulfilled")
|
||||
|
||||
this.logger.info("Updates with succes: " + success.length);
|
||||
|
||||
values.forEach(promise => {
|
||||
this.logger.info("Promise finish with status: " + promise.status)
|
||||
|
||||
if (promise.status === "rejected") {
|
||||
this.logger.error("reject with: " + JSON.stringify(promise.reason || {}));
|
||||
throw new Error(promise.reason );
|
||||
}
|
||||
|
||||
if (promise.status === "fulfilled") {
|
||||
this.logger.info("success with: " + JSON.stringify(promise.value || {}));
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
|
||||
const jobsUpdated = [];
|
||||
|
||||
for (const [index, table] of updateInputDTO.tables.entries()) {
|
||||
const id = `${pipelineIdFormat}_${index}`;
|
||||
this.logger.info('Updating input reference for table', table.name);
|
||||
const body = {}
|
||||
const jobUpdate = {
|
||||
job_id: `${pipelineId}_${index}`,
|
||||
}
|
||||
|
||||
if (table.type !== "incremental_with_qualify") {
|
||||
delete table.destinations?.qualify;
|
||||
}
|
||||
|
||||
if (table.memory) {
|
||||
jobUpdate["memory"] = {
|
||||
amount: table.memory * 1000
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Updating input reference for table: ' + table.name);
|
||||
let hasUpdateSyncMode = false;
|
||||
|
||||
const jobSyncMode = {}
|
||||
|
||||
if (table.columns) {
|
||||
body['column_include_list'] = table.columns;
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['column_include_list'] = table.columns;
|
||||
}
|
||||
|
||||
if (table.reference_column) {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['incremental_column_name'] = table.reference_column.name;
|
||||
jobSyncMode['incremental_column_type'] = table.reference_column.type;
|
||||
}
|
||||
|
||||
if (table.identifier_columns) {
|
||||
body['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
this.logger.info('Request body', body);
|
||||
const updateCollumns = this.platformAPI.proxy(
|
||||
'PATCH',
|
||||
`/jobs/${id}/input`,
|
||||
user,
|
||||
body
|
||||
)
|
||||
requests.push(updateCollumns);
|
||||
|
||||
if (table.memory) {
|
||||
this.logger.info('Updating memory allocation for table', table.name);
|
||||
const updateMemory = this.platformAPI.proxy(
|
||||
'PUT',
|
||||
`/jobs/${id}/memory`,
|
||||
user,
|
||||
{
|
||||
amount: table.memory
|
||||
}
|
||||
)
|
||||
requests.push(updateMemory);
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
if (table.type) {
|
||||
const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
|
||||
requests.push(updateSyncMode);
|
||||
hasUpdateSyncMode = true;
|
||||
jobSyncMode['target_load_type'] = table.type;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Create Platform Request for each JOB');
|
||||
if(hasUpdateSyncMode) {
|
||||
jobUpdate["sync_mode"] = jobSyncMode;
|
||||
}
|
||||
|
||||
if (updateInputDTO.cron) {
|
||||
const crnUpdatedRequest = new Promise(async (resolve, reject) => {
|
||||
const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
|
||||
|
||||
if (response.error) {
|
||||
this.logger.error('Error updating pipeline cron', response.error);
|
||||
return reject(new ErrorBuilder(response.error));
|
||||
if (Object.keys(table.destinations).length > 1) {
|
||||
let hasChanges = false
|
||||
const jobRenameTables = {
|
||||
raw: {},
|
||||
qualify: {}
|
||||
}
|
||||
this.logger.error('Pipeline cron updated successfully', response);
|
||||
return resolve(response);
|
||||
});
|
||||
requests.push(crnUpdatedRequest);
|
||||
|
||||
if (Object.keys(table.destinations.raw).length > 1) {
|
||||
hasChanges = true;
|
||||
jobRenameTables.raw = table.destinations.raw;
|
||||
}
|
||||
|
||||
if (Object.keys(table.destinations.qualify).length > 1) {
|
||||
hasChanges = true;
|
||||
jobRenameTables.qualify = table.destinations.qualify;
|
||||
}
|
||||
|
||||
if (hasChanges) {
|
||||
jobUpdate['rename_tables'] = jobRenameTables;
|
||||
}
|
||||
}
|
||||
|
||||
jobsUpdated.push(jobUpdate);
|
||||
}
|
||||
|
||||
this.logger.info('Executing all request for the platform api');
|
||||
this.logger.info('Request body:' + JSON.stringify({
|
||||
jobs_updated: jobsUpdated
|
||||
}));
|
||||
|
||||
const results = await Promise.allSettled(requests);
|
||||
this.logger.info('Platform api response', results);
|
||||
|
||||
return updateInputResponse;
|
||||
|
||||
}
|
||||
|
||||
private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) {
|
||||
const body = {
|
||||
target_load_type: table.type
|
||||
}
|
||||
|
||||
if (table.type === 'incremental_with_qualify') {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
body['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
if (table.type === 'incremental') {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
}
|
||||
|
||||
this.logger.info('Updating pipeline sync mode', {
|
||||
pipelineId,
|
||||
body
|
||||
});
|
||||
|
||||
return this.platformAPI.proxy(
|
||||
"POST",
|
||||
`/jobs/jdbc/${pipelineId}/sync-mode`,
|
||||
const response = await this.platformAPI.proxy(
|
||||
'PUT',
|
||||
`/pipeline/${pipelineId}/jobs`,
|
||||
user,
|
||||
body
|
||||
{
|
||||
job_updates: jobsUpdated
|
||||
}
|
||||
)
|
||||
this.logger.info('Platform api response: ' + JSON.stringify(response));
|
||||
}
|
||||
|
||||
private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
|
||||
try {
|
||||
const response = await this.platformAPI.proxy(
|
||||
'PATCH',
|
||||
`/pipeline/${pipelineId}`,
|
||||
user,
|
||||
{
|
||||
cron
|
||||
}
|
||||
);
|
||||
|
||||
return response
|
||||
} catch (error) {
|
||||
return {
|
||||
error: error.message
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -845,6 +845,24 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Post('pipeline/:pipelineId/pipeline_run/:runId/cancel')
|
||||
@ApiOperation({ summary: 'Cancel a running pipeline run' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async cancelPipelineRun(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('runId') runId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const normalizedRunId = this.normalizePipelineId(runId);
|
||||
|
||||
return this.platformApiService.proxy(
|
||||
'POST',
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/cancel`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== JOBS - COLUMN EDITING ROUTES ====================
|
||||
|
||||
@Put('jobs/:jobId/input')
|
||||
@@ -944,6 +962,61 @@ export class PlatformApiController {
|
||||
);
|
||||
}
|
||||
|
||||
@Delete('jobs/:jobId')
|
||||
@ApiOperation({ summary: 'Delete a job and mark its table as deleted in DynamoDB' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||
async deleteJob(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedJobId = this.normalizeJobId(jobId);
|
||||
const esPipelineId = this.extractPipelineIdFromJobId(jobId);
|
||||
|
||||
// Fetch job details and ES pipeline in parallel — both are reads with no mutual dependency
|
||||
const [jobResult, esPipelineResult] = await Promise.allSettled([
|
||||
this.platformApiService.proxy('GET', `/jobs/${normalizedJobId}`, user),
|
||||
this.elasticsearchService.getPipeline(user.customer_name, esPipelineId),
|
||||
]);
|
||||
|
||||
const tableName: string | undefined = jobResult.status === 'fulfilled'
|
||||
? jobResult.value?.source_config?.table_name
|
||||
: undefined;
|
||||
|
||||
if (jobResult.status === 'rejected') {
|
||||
this.logger.warn('deleteJob: could not fetch job before deletion', { jobId, error: jobResult.reason?.message });
|
||||
}
|
||||
|
||||
// Main operation
|
||||
const result = await this.platformApiService.proxy('DELETE', `/jobs/${normalizedJobId}`, user);
|
||||
|
||||
// Sync deleted status to DynamoDB — follows syncJobInputToDynamoDB pattern
|
||||
// config.tables in ES stores the DynamoDB input ID (legacy field naming)
|
||||
const dynamoInputId: string | undefined = esPipelineResult.status === 'fulfilled'
|
||||
? esPipelineResult.value?.config?.tables
|
||||
: undefined;
|
||||
|
||||
if (esPipelineResult.status === 'rejected') {
|
||||
this.logger.warn('deleteJob: could not fetch ES pipeline for DynamoDB sync', { jobId, error: esPipelineResult.reason?.message });
|
||||
}
|
||||
|
||||
if (tableName && dynamoInputId) {
|
||||
await this.dynamoDBService.updateInputTable(
|
||||
user.customer_id,
|
||||
dynamoInputId,
|
||||
tableName,
|
||||
{ status: 'deleted' },
|
||||
).catch((error) => {
|
||||
this.logger.error('Failed to sync job deletion to DynamoDB', { jobId, tableName, error: error.message });
|
||||
});
|
||||
} else if (!tableName) {
|
||||
this.logger.warn('deleteJob: table name not resolved, skipping DynamoDB sync', { jobId });
|
||||
} else {
|
||||
this.logger.warn('deleteJob: no input ID in ES pipeline, skipping DynamoDB sync', { jobId });
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
||||
|
||||
@Get('jobs/jdbc/:jobId')
|
||||
|
||||
@@ -8,9 +8,10 @@ import { ElasticsearchModule } from '../../services/elasticsearch';
|
||||
import { DynamoDBModule } from '../../services/dynamodb';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
import { CatalogModule } from '../catalog/catalog.module';
|
||||
import { PipelinesV2Module } from '../pipelinesV2/pipelines.module';
|
||||
|
||||
@Module({
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule],
|
||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, PipelinesV2Module],
|
||||
controllers: [PlatformApiController],
|
||||
providers: [PlatformApiService, DadosferaLogger],
|
||||
exports: [PlatformApiService],
|
||||
|
||||
@@ -89,12 +89,12 @@ export class PlatformApiService {
|
||||
|
||||
// Propagate non-2xx responses as HttpExceptions
|
||||
if (response.status >= 400) {
|
||||
this.logger.error('Platform API upstream error', {
|
||||
this.logger.error('Platform API upstream error' + JSON.stringify({
|
||||
status: response.status,
|
||||
data: response.data,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
}));
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
@@ -107,6 +107,8 @@ export class PlatformApiService {
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
this.logger.error(error)
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ export interface InputDocument {
|
||||
type: string;
|
||||
columns?: string[];
|
||||
reference_column?: ReferenceColumn;
|
||||
status?: string;
|
||||
}>;
|
||||
credentials?: Record<string, any>;
|
||||
}
|
||||
@@ -125,9 +126,13 @@ export class DynamoDBService {
|
||||
},
|
||||
});
|
||||
|
||||
const { Item } = await this.documentClient.send(getCommand);
|
||||
|
||||
return Item as InputDocument | null;
|
||||
try {
|
||||
const { Item } = await this.documentClient.send(getCommand);
|
||||
return Item as InputDocument | null;
|
||||
} catch (error) {
|
||||
this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async deleteInput(clientId: string, inputId: string): Promise<void> {
|
||||
@@ -163,6 +168,7 @@ export class DynamoDBService {
|
||||
type?: string;
|
||||
columns?: string[];
|
||||
reference_column?: ReferenceColumn | null;
|
||||
status?: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
const dynamoTableName = DYNAMODB_CONFIG.inputsTable();
|
||||
@@ -203,6 +209,7 @@ export class DynamoDBService {
|
||||
updatedTable.reference_column = changes.reference_column;
|
||||
}
|
||||
}
|
||||
if ('status' in changes) updatedTable.status = changes.status;
|
||||
|
||||
tables[tableIndex] = updatedTable;
|
||||
|
||||
@@ -231,4 +238,53 @@ export class DynamoDBService {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a specific table entry from the input document.
|
||||
* Fetches the current document, removes the matching table, and saves.
|
||||
*/
|
||||
async removeInputTable(
|
||||
clientId: string,
|
||||
inputId: string,
|
||||
tableName: string,
|
||||
): Promise<void> {
|
||||
const dynamoTableName = DYNAMODB_CONFIG.inputsTable();
|
||||
|
||||
this.logger.info('DynamoDB: Removing input table', {
|
||||
inputId,
|
||||
tableName,
|
||||
});
|
||||
|
||||
const current = await this.findInput(clientId, inputId);
|
||||
if (!current) {
|
||||
this.logger.warn('DynamoDB: Input not found for table removal', { inputId });
|
||||
return;
|
||||
}
|
||||
|
||||
const tables = (current.tables || []).filter((t) => t.name !== tableName);
|
||||
|
||||
const putCommand = new PutCommand({
|
||||
TableName: dynamoTableName,
|
||||
Item: {
|
||||
...current,
|
||||
tables,
|
||||
updated_at: new Date().toISOString(),
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
await this.documentClient.send(putCommand);
|
||||
this.logger.info('DynamoDB: Input table removed successfully', {
|
||||
inputId,
|
||||
tableName,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('DynamoDB: Failed to remove input table', {
|
||||
inputId,
|
||||
tableName,
|
||||
error: error.message,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { NimbusService } from "./nimbus.service";
|
||||
import DadosferaLogger from "@dadosfera/dadosfera-logs";
|
||||
|
||||
@Module({
|
||||
providers: [NimbusService, DadosferaLogger],
|
||||
exports: [NimbusService],
|
||||
})
|
||||
export class NimbusServicesModule {}
|
||||
@@ -0,0 +1,56 @@
|
||||
import DadosferaLogger from "@dadosfera/dadosfera-logs";
|
||||
import { Inject, Injectable } from "@nestjs/common";
|
||||
import axios from "axios";
|
||||
|
||||
type TableUpdate = {
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class NimbusService {
|
||||
private logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
private buildUrl(customerName: string) {
|
||||
if (process.env.ENV === 'prd') {
|
||||
return `https://nimbus-${customerName}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
return `https://nimbus-${customerName}.${process.env.ENV.replace(
|
||||
'local',
|
||||
'stg',
|
||||
)}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
async renameTable(customerName: string, database: string, old: TableUpdate, update: TableUpdate) {
|
||||
const nimbusUrl = this.buildUrl(customerName);
|
||||
|
||||
const path = `/api/catalog/rename-tables/?database_name=${encodeURIComponent(database)}&table_name=${encodeURIComponent(old.table_name)}&table_schema=${encodeURIComponent(old.table_schema)}`;
|
||||
|
||||
try {
|
||||
this.logger.info("Request for PATCH " + nimbusUrl + path);
|
||||
this.logger.info("Payload: " + JSON.stringify(update));
|
||||
const { data } = await axios.patch(nimbusUrl + path, {
|
||||
table_name: update.table_name,
|
||||
table_schema: update.table_schema
|
||||
})
|
||||
|
||||
return data;
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
return {
|
||||
message: error.message,
|
||||
database,
|
||||
old,
|
||||
update
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user