mirror of
https://github.com/dadosfera/maestro.git
synced 2026-08-31 19:58:21 +00:00
FEAT: Creat a Upload CSV Pipeline
This commit is contained in:
@@ -277,13 +277,15 @@ export class PipelinesController {
|
||||
|
||||
const { sep, header, encoding, description } = body;
|
||||
|
||||
const p = {
|
||||
const [file_name, file_format] = file.originalname.split('.');
|
||||
|
||||
const upload_pipeline = {
|
||||
connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
|
||||
connector_name: 'Amazon S3',
|
||||
connector_plugin: 'aws_s3',
|
||||
connector_version: '1.0.0',
|
||||
image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
|
||||
name: body.name || file.originalname.split('.')[0],
|
||||
name: body.name || file_name,
|
||||
description,
|
||||
transformations_ids: [],
|
||||
tags: [],
|
||||
@@ -292,16 +294,18 @@ export class PipelinesController {
|
||||
properties: {
|
||||
engine: 'csv',
|
||||
source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
|
||||
source_prefix: name,
|
||||
source_prefix: `${user.customer_name}/${name}.${file_format}`,
|
||||
file_format_params: { sep, encoding, header: Boolean(header) },
|
||||
is_a_upload_csv: true,
|
||||
},
|
||||
input_id: undefined,
|
||||
};
|
||||
|
||||
const pipeline = await this.pipelinesClientService.create(p, metadata);
|
||||
console.log(pipeline);
|
||||
const pipeline = await this.pipelinesClientService.create(
|
||||
upload_pipeline,
|
||||
metadata,
|
||||
);
|
||||
|
||||
return response;
|
||||
return pipeline;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user