gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-11-08 02:09:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

FIX: create pipeline from any file type on upload

Browse Source
This commit is contained in:
Gabriel Amorim
2023-02-22 10:18:28 -03:00
parent ee8c5e7a93
commit ac3132b6eb
5 changed files with 42 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docsfera.json
+14 -2
View File
@@ -4676,7 +4676,7 @@
}
},
"info": {
"title": "Maestro - feat/docs-refactor",
"title": "Maestro - fix/upload-files",
"description": "This is the Maestro API",
"version": "1.0.0",
"contact": {}
@@ -6346,6 +6346,9 @@
"description": {
"type": "string"
},
"type": {
"type": "string"
},
"connection_id": {
"type": "string"
},
@@ -6386,6 +6389,7 @@
"required": [
"name",
"description",
"type",
"connection_id",
"cron",
"connector_name",
@@ -6406,6 +6410,9 @@
"description": {
"type": "string"
},
"type": {
"type": "string"
},
"connection_id": {
"type": "string"
},
@@ -6453,6 +6460,7 @@
"id",
"name",
"description",
"type",
"connection_id",
"cron",
"connector_name",
@@ -6520,6 +6528,9 @@
},
"header": {
"type": "boolean"
},
"engine": {
"type": "string"
}
},
"required": [
@@ -6528,7 +6539,8 @@
"description",
"encoding",
"sep",
"header"
"header",
"engine"
]
},
"IRole": {
package-lock.json
Generated
+7 -7
View File
@@ -12,7 +12,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.29.0",
"@dadosfera/protospack-v2": "^3.29.1-beta.1",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.29.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz",
"integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==",
"version": "3.29.1-beta.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1-beta.1.tgz",
"integrity": "sha512-CzHoIp052WcBYPJAXEJeZ4+2lwjpa9QuPDrJpLZtvAtwKcN7fnXDQyECVWh7loZ+p5LfsosE8lrhLMZCe0pRVQ==",
"dependencies": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -12284,9 +12284,9 @@
}
},
"@dadosfera/protospack-v2": {
"version": "3.29.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz",
"integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==",
"version": "3.29.1-beta.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1-beta.1.tgz",
"integrity": "sha512-CzHoIp052WcBYPJAXEJeZ4+2lwjpa9QuPDrJpLZtvAtwKcN7fnXDQyECVWh7loZ+p5LfsosE8lrhLMZCe0pRVQ==",
"requires": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
package.json
+2 -2
View File
@@ -29,7 +29,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.29.0",
"@dadosfera/protospack-v2": "^3.29.1-beta.1",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -96,4 +96,4 @@
"tsconfig-paths": "^3.14.1",
"typescript": "^4.6.3"
}
}
}
src/modules/pipelinesV2/interfaces.ts
+5
View File
@@ -10,6 +10,8 @@ export class IPipelineV2 {
@ApiProperty()
description: string;
@ApiProperty()
type: string;
@ApiProperty()
connection_id: string;
@ApiProperty()
cron: string;
@@ -104,4 +106,7 @@ export class ICreatePipelineCSVFile {
@ApiProperty()
header: boolean;
@ApiProperty()
engine: string;
}
src/modules/pipelinesV2/pipelines.controller.ts
+14 -7
View File
@@ -370,13 +370,21 @@ export class PipelinesController {
@User() user: RequestUser,
@Body() body: ICreatePipelineCSVFile,
) {
this.logger.info('/upload - Upload Connector Route');
this.logger.info('/file - Upload Connector Route');
const metadata = PackTheMetadata({ ...user });
const { name, sep, header, encoding, description, file_name } = body;
const { name, sep, header, encoding, description, file_name, engine } =
body;
const file_format_params = {
...(sep && { sep }),
...(encoding && { encoding }),
...(header && { header }),
};
const source_prefix = `${user.customer_name}/${file_name}`;
const upload_pipeline = {
const upload_pipeline: ICreatePipelineV2Req = {
connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
connector_name: 'Amazon S3',
connector_plugin: 'aws_s3',
@@ -387,13 +395,12 @@ export class PipelinesController {
transformations_ids: [],
tags: [],
cron: '@once',
config: { cron: '@once', tables: [] },
type: 'upload',
properties: {
engine: 'csv',
engine,
source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
source_prefix,
file_format_params: { sep, encoding, header: Boolean(header) },
is_a_upload_csv: true,
file_format_params,
},
input_id: undefined,
};