FEAT: update nimbus after input

This commit is contained in:
marcos-silva-rodrigues
2026-03-24 12:28:20 -03:00
parent 959210e354
commit 21c57e5620
10 changed files with 140 additions and 46 deletions
+14 -7
View File
@@ -17,6 +17,7 @@ import {
InputCreateGenericRequest,
InputCreateS3Request,
InputNewCreateRequest,
InputUpdateResponse,
TestConnectionRequest,
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -71,8 +72,8 @@ export class InputsService {
objectCamelToSnake(createInputResponse);
return createInputResponse;
},
update: async (updateInputDTO: UpdateInputRequest) => {
this.logger.info('InputClientService - Update');
update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
const updateInputResponse = await lastValueFrom(
this.inputWriteService.InputUpdate(updateInputDTO),
);
@@ -207,16 +208,22 @@ export class InputsService {
async update(id: string, data, info: Info) {
// this.validateCron({ ...data, info });
try {
const updateInputResponse: any = await this.OLD_inputClient.update({
const {
nimbusUpdates,
input
} = await this.OLD_inputClient.update({
id,
info,
...data,
info,
});
updateInputResponse.input = this.adjustInputPayload(
updateInputResponse?.input,
const updateInputResponse = this.adjustInputPayload(
input,
);
return updateInputResponse;
return {
input: updateInputResponse,
nimbusUpdates
};
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
@@ -294,13 +294,14 @@ export class PipelinesController {
) {
this.logger.info('PipelinesController - update', { user });
const { customer_id, customer_name, user_id, username } = user;
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
customer_id: user.customer_id,
pipeline_id: pipelineId
};
const metadata = PackTheMetadata({
customer_id,
customer_name,
+5 -2
View File
@@ -12,6 +12,8 @@ import { ConnectorModule } from '../connector/connector.module';
import { InputsModule } from '../inputs/inputs.module';
import { TransformationsModule } from '../transformations/transformations.module';
import { PlatformApiModule } from '../platform-api/platform-api.module';
import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
import { NimbusService } from 'src/services/nimbus/nimbus.service';
const client = new PipelinesClientConfiguration();
@@ -22,10 +24,11 @@ const client = new PipelinesClientConfiguration();
ConnectorModule,
InputsModule,
TransformationsModule,
PlatformApiModule
PlatformApiModule,
NimbusServicesModule
],
controllers: [PipelinesController],
providers: [PipelinesService, DadosferaLogger],
providers: [PipelinesService, DadosferaLogger, NimbusService],
exports: [PipelinesService],
})
export class PipelinesV2Module {}
+39 -10
View File
@@ -28,7 +28,10 @@ import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import { PlatformApiService } from '../platform-api/platform-api.service';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { Info, NewTable } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { NimbusUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import axios from 'axios';
import { NimbusService } from 'src/services/nimbus/nimbus.service';
export class PipelinesService implements OnModuleInit {
logger: DadosferaLogger;
@@ -42,7 +45,8 @@ export class PipelinesService implements OnModuleInit {
private readonly connectorService: ConnectorClientService,
private readonly inputsService: InputsService,
private readonly transformationsService: TransformationsService,
private readonly platformAPI: PlatformApiService
private readonly platformAPI: PlatformApiService,
private readonly nimbusService: NimbusService
) {
this.logger = dadosferaLogger.logger;
}
@@ -350,18 +354,46 @@ export class PipelinesService implements OnModuleInit {
this.logger.info('Update Dynamo Reference');
const pipelineIdFormat = pipelineId.split('-').join('_');
console.log("updatePipelineInput", info, updateInputDTO)
const updateInputResponse = await this.inputsService.update(
inputId,
updateInputDTO,
info
)
this.logger.info('Dynamo Response', updateInputResponse);
this.logger.info(JSON.stringify(updateInputResponse))
await this.updateNimbus(info.customer, updateInputResponse.nimbusUpdates);
await this.updatePlatformJobs(
pipelineIdFormat,
updateInputDTO,
user
);
return updateInputResponse;
}
private async updateNimbus(customer: string, changes: NimbusUpdate[]) {
this.logger.info('Nimbus Changes: ' + JSON.stringify(changes));
if(!changes || changes.length === 0) return;
const requests = changes.map(change => {
return this.nimbusService.renameTable(customer, change.database, change.old, change.new);
})
const values = await Promise.allSettled(requests);
const success = values.map(request => request.status === "fulfilled")
this.logger.info("Updates with succes: " + success.length);
}
async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
const jobsUpdated = [];
for (const [index, table] of updateInputDTO.tables.entries()) {
const jobUpdate = {
job_id: `${pipelineIdFormat}_${index}`,
job_id: `${pipelineId}_${index}`,
}
if (table.memory) {
@@ -370,7 +402,7 @@ export class PipelinesService implements OnModuleInit {
}
}
this.logger.info('Updating input reference for table', table.name);
this.logger.info('Updating input reference for table: ' + table.name);
let hasUpdateSyncMode = false;
const jobSyncMode = {}
@@ -431,17 +463,14 @@ export class PipelinesService implements OnModuleInit {
}));
const response = await this.platformAPI.proxy(
'POST',
`/pipelines/${pipelineIdFormat}/batch-update`,
'PUT',
`/pipeline/${pipelineId}/jobs`,
user,
{
job_updates: jobsUpdated
}
)
this.logger.info('Platform api response: ' + JSON.stringify(response));
return updateInputResponse;
}
}
@@ -89,12 +89,12 @@ export class PlatformApiService {
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
this.logger.error('Platform API upstream error', {
this.logger.error('Platform API upstream error' + JSON.stringify({
status: response.status,
data: response.data,
path,
method: method.toUpperCase(),
});
}));
throw new HttpException(response.data, response.status);
}
@@ -107,6 +107,8 @@ export class PlatformApiService {
method: method.toUpperCase(),
});
this.logger.error(error)
if (error instanceof HttpException) {
throw error;
}