Files
maestro/src/modules/pipelinesV2/pipelines.service.ts
T

481 lines
14 KiB
TypeScript

/* eslint-disable no-async-promise-executor */
import {
BadRequestException,
HttpException,
HttpStatus,
Inject,
OnModuleInit,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
Messages,
WriteService,
ReadService,
ProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { lastValueFrom } from 'rxjs';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service';
import { InputsService } from '../inputs/inputs.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { TransformationsService } from '../transformations/transformations.service';
import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import { PlatformApiService } from '../platform-api/platform-api.service';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
export class PipelinesService implements OnModuleInit {
logger: DadosferaLogger;
pipelineWriteService: WriteService.PipelineV2WriteService;
pipelineReadService: ReadService.PipelineV2ReadService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly connectorService: ConnectorClientService,
private readonly inputsService: InputsService,
private readonly transformationsService: TransformationsService,
private readonly platformAPI: PlatformApiService
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.pipelineReadService =
this.grpcClient.getService<ReadService.PipelineV2ReadService>(
ProtoServices.PipelineV2ReadService,
);
this.pipelineWriteService =
this.grpcClient.getService<WriteService.PipelineV2WriteService>(
ProtoServices.PipelineV2WriteService,
);
}
adjustInputPayload(payload) {
const { input } = payload;
if (input) {
if (input.input_generic) {
const { credentials } = input.input_generic;
return {
...input.input_generic,
credentials: credentials && JSON.parse(credentials),
};
}
if (input.input_s3) return input.input_s3;
if (input.input_jdbc) return input.input_jdbc;
const { credentials } = input;
return {
...input,
credentials: credentials && JSON.parse(credentials),
};
}
return input;
}
async create(
body: ICreatePipelineV2Req,
metadata,
): Promise<Messages.PipelineV2CreateResponse> {
this.logger.info('PipelinesV2ClientService - Create');
const pipelineV2CreateRequest: PipelineV2CreateRequest = {
...body,
input_id: body.input_id,
transformations_ids: body.transformations_ids,
tags: body.tags,
properties: body.properties && JSON.stringify(body.properties),
};
const createPipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2Create(
pipelineV2CreateRequest,
metadata,
),
).catch((error: { details: string }) => {
if (error.details.includes('INVALID_REQUEST')) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [errorType, message] = error.details.split('|');
throw new BadRequestException(message);
}
throw new ErrorBuilder(error.details);
});
this.logger.info('Done');
return createPipelineResponse;
}
async findAll(
data: Messages.PipelineV2FindAllRequest,
metadata,
): Promise<Messages.PipelineV2FindAllResponse> {
this.logger.info('PipelinesClientService - FindAll');
const findAllPipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2FindAll(data, metadata),
)
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return findAllPipelineResponse;
}
async findOne(
data: Messages.PipelineV2FindOneRequest,
metadata,
): Promise<Messages.PipelineV2FindOneResponse> {
this.logger.info('PipelinesClientService - FindOne');
const findOnePipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2FindOne(data, metadata),
);
console.log('pipeline find one response', findOnePipelineResponse);
this.logger.info('Done');
return findOnePipelineResponse;
}
async update(
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
metadata,
) {
this.logger.info('PipelinesClientService - Update');
const updatePipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2Update(
UpdatePipelineRequest,
metadata,
),
);
this.logger.info('Done');
return updatePipelineResponse;
}
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
const { id, metadata, user } = data;
const info = {
user_id: user.user_id,
customer_id: user.customer_id,
customer: user.customer_name,
};
let existsInProduct = true;
const { pipeline } = await lastValueFrom(
this.pipelineReadService.PipelineV2FindOne({ id }, metadata),
).catch((e) => {
if (e.details === ErrorCodes.PIPELINE.NOT_FOUND) existsInProduct = false;
else throw new ErrorBuilder(e.details);
return { pipeline: undefined };
});
await lastValueFrom(
this.pipelineWriteService.PipelineV2Remove({ id }, metadata),
);
if (existsInProduct) {
//{pipeline:{tables: {tables: [], input_id: ''}}}
const input = pipeline.config.tables
? JSON.parse(pipeline.config.tables)
: null;
if (input)
await this.inputsService
.remove({
id: input.input_id,
info,
})
.catch((error) =>
this.logger.error('Could not delete input', {
data: { input, error },
}),
);
const transformations: { id: string }[] = pipeline.transformations
? JSON.parse(pipeline.transformations)
: null;
if (transformations && transformations.length)
for (const transformation of transformations) {
await this.transformationsService
.remove({
id: transformation.id,
info,
})
.catch((error) =>
this.logger.error('Could not delete transformation', {
data: { transformation, error },
}),
);
}
}
}
async findOneProperties(id: string, metadata: Metadata) {
this.logger.info('PipelinesClientService - findOneProperties');
const { pipeline } = await this.findOne({ id }, metadata);
const properties = JSON.parse(pipeline.properties);
const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`;
const res = await this.connectorService.getConnectorDetails(
connectorId,
metadata.get('language')[0].toString(),
);
const connector: { [key: string]: any; config_controls: any[] } =
JSON.parse(res.connector);
const config_controls = (connector.config_controls || []).filter(
(config_control) => {
const path: string = config_control.name;
if (!objHasPath({ object: { properties }, path })) return false;
let default_value = getObjValueFromPath({
object: { properties },
path,
});
if (default_value != undefined) {
if (
path === 'properties.spreadsheet_id' &&
typeof default_value === 'string'
)
default_value = 'https://docs.google.com/spreadsheets/d/'.concat(
default_value,
);
config_control.default_value = default_value;
}
return true;
},
);
return {
pipeline: {
...pipeline,
config_controls,
properties,
},
};
}
async findOneObjects(id: string, metadata: Metadata) {
this.logger.info('PipelinesClientService - FindOneObjects');
const { objects } = await lastValueFrom(
this.pipelineReadService.PipelineV2FindObjects({ id }, metadata),
);
this.logger.info('Done');
return { objects: JSON.parse(objects) };
}
async initUploadFile(uploadFile, metadata) {
const body: Messages.PipelineV2UploadFileRequest = {
name: uploadFile.name,
parts: uploadFile.parts,
};
return lastValueFrom(
this.pipelineWriteService.PipelineV2UploadFile(body, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async downloadLogs(pipeline_run_id: string, metadata: Metadata) {
this.logger.info('PipelinesClientService - DownloadLogs');
const { url } = await lastValueFrom(
this.pipelineReadService.PipelineV2DownloadLogs(
{ pipeline_run_id },
metadata,
),
);
this.logger.info('Done');
return url;
}
async completeUploadFile(uploadFile, metadata) {
const body: Messages.PipelineV2CompleteUploadFileRequest = {
upload_id: uploadFile.upload_id,
parts: JSON.stringify(uploadFile.parts),
name: uploadFile.file_name,
};
return lastValueFrom(
this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
async getMonitoringDashboardUrl(metadata: Metadata) {
this.logger.info('PipelinesClientService - getMonitoringDashboardUrl');
const res = await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: '95',
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata,
),
);
this.logger.info('Done');
return res;
}
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
this.logger.info('InputClientService - Update');
this.logger.info('Update Dynamo Reference');
const pipelineIdFormat = pipelineId.split('-').join('_');
const updateInputResponse = await this.inputsService.update(
inputId,
updateInputDTO,
info
)
const requests = [];
this.logger.info('Dynamo Response', updateInputResponse);
for (const [index, table] of updateInputDTO.tables.entries()) {
const id = `${pipelineIdFormat}_${index}`;
this.logger.info('Updating input reference for table', table.name);
const body = {}
if (table.columns) {
body['column_include_list'] = table.columns;
}
if (table.reference_column) {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
}
if (table.identifier_columns) {
body['primary_keys'] = table.identifier_columns;
}
this.logger.info('Request body', body);
const updateCollumns = this.platformAPI.proxy(
'PATCH',
`/jobs/${id}/input`,
user,
body
)
requests.push(updateCollumns);
if (table.memory) {
this.logger.info('Updating memory allocation for table', table.name);
const updateMemory = this.platformAPI.proxy(
'PUT',
`/jobs/${id}/memory`,
user,
{
amount: table.memory
}
)
requests.push(updateMemory);
}
if (table.type) {
const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
requests.push(updateSyncMode);
}
}
this.logger.info('Create Platform Request for each JOB');
if (updateInputDTO.cron) {
const crnUpdatedRequest = new Promise(async (resolve, reject) => {
const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
if (response.error) {
this.logger.error('Error updating pipeline cron', response.error);
return reject(new ErrorBuilder(response.error));
}
this.logger.error('Pipeline cron updated successfully', response);
return resolve(response);
});
requests.push(crnUpdatedRequest);
}
this.logger.info('Executing all request for the platform api');
const results = await Promise.allSettled(requests);
this.logger.info('Platform api response', results);
return updateInputResponse;
}
private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) {
const body = {
target_load_type: table.type
}
if (table.type === 'incremental_with_qualify') {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
body['primary_keys'] = table.identifier_columns;
}
if (table.type === 'incremental') {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
}
this.logger.info('Updating pipeline sync mode', {
pipelineId,
body
});
return this.platformAPI.proxy(
"POST",
`/jobs/jdbc/${pipelineId}/sync-mode`,
user,
body
)
}
private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
try {
const response = await this.platformAPI.proxy(
'PATCH',
`/pipeline/${pipelineId}`,
user,
{
cron
}
);
return response
} catch (error) {
return {
error: error.message
}
}
}
}