From ac3132b6eb5fa95dbcddc9afba12b26f6d1a5fb9 Mon Sep 17 00:00:00 2001 From: Gabriel Amorim Date: Wed, 22 Feb 2023 10:18:28 -0300 Subject: [PATCH] FIX: create pipeline from any file type on upload --- docsfera.json | 16 ++++++++++++-- package-lock.json | 14 ++++++------- package.json | 4 ++-- src/modules/pipelinesV2/interfaces.ts | 5 +++++ .../pipelinesV2/pipelines.controller.ts | 21 ++++++++++++------- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/docsfera.json b/docsfera.json index dee5148..44f68f2 100644 --- a/docsfera.json +++ b/docsfera.json @@ -4676,7 +4676,7 @@ } }, "info": { - "title": "Maestro - feat/docs-refactor", + "title": "Maestro - fix/upload-files", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} @@ -6346,6 +6346,9 @@ "description": { "type": "string" }, + "type": { + "type": "string" + }, "connection_id": { "type": "string" }, @@ -6386,6 +6389,7 @@ "required": [ "name", "description", + "type", "connection_id", "cron", "connector_name", @@ -6406,6 +6410,9 @@ "description": { "type": "string" }, + "type": { + "type": "string" + }, "connection_id": { "type": "string" }, @@ -6453,6 +6460,7 @@ "id", "name", "description", + "type", "connection_id", "cron", "connector_name", @@ -6520,6 +6528,9 @@ }, "header": { "type": "boolean" + }, + "engine": { + "type": "string" } }, "required": [ @@ -6528,7 +6539,8 @@ "description", "encoding", "sep", - "header" + "header", + "engine" ] }, "IRole": { diff --git a/package-lock.json b/package-lock.json index 8cae407..bb34985 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.29.0", + "@dadosfera/protospack-v2": "^3.29.1-beta.1", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -1727,9 +1727,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.29.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz", - "integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==", + "version": "3.29.1-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1-beta.1.tgz", + "integrity": "sha512-CzHoIp052WcBYPJAXEJeZ4+2lwjpa9QuPDrJpLZtvAtwKcN7fnXDQyECVWh7loZ+p5LfsosE8lrhLMZCe0pRVQ==", "dependencies": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", @@ -12284,9 +12284,9 @@ } }, "@dadosfera/protospack-v2": { - "version": "3.29.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz", - "integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==", + "version": "3.29.1-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1-beta.1.tgz", + "integrity": "sha512-CzHoIp052WcBYPJAXEJeZ4+2lwjpa9QuPDrJpLZtvAtwKcN7fnXDQyECVWh7loZ+p5LfsosE8lrhLMZCe0pRVQ==", "requires": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", diff --git a/package.json b/package.json index b829842..0dcc508 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.29.0", + "@dadosfera/protospack-v2": "^3.29.1-beta.1", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -96,4 +96,4 @@ "tsconfig-paths": "^3.14.1", "typescript": "^4.6.3" } -} \ No newline at end of file +} diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 9cdd9dd..c9d23d3 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -10,6 +10,8 @@ export class IPipelineV2 { @ApiProperty() description: string; @ApiProperty() + type: string; + @ApiProperty() connection_id: string; @ApiProperty() cron: string; @@ -104,4 +106,7 @@ export class ICreatePipelineCSVFile { @ApiProperty() header: boolean; + + @ApiProperty() + engine: string; } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index c457f33..6c53860 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -370,13 +370,21 @@ export class PipelinesController { @User() user: RequestUser, @Body() body: ICreatePipelineCSVFile, ) { - this.logger.info('/upload - Upload Connector Route'); + this.logger.info('/file - Upload Connector Route'); const metadata = PackTheMetadata({ ...user }); - const { name, sep, header, encoding, description, file_name } = body; + const { name, sep, header, encoding, description, file_name, engine } = + body; + + const file_format_params = { + ...(sep && { sep }), + ...(encoding && { encoding }), + ...(header && { header }), + }; + const source_prefix = `${user.customer_name}/${file_name}`; - const upload_pipeline = { + const upload_pipeline: ICreatePipelineV2Req = { connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION, connector_name: 'Amazon S3', connector_plugin: 'aws_s3', @@ -387,13 +395,12 @@ export class PipelinesController { transformations_ids: [], tags: [], cron: '@once', - config: { cron: '@once', tables: [] }, + type: 'upload', properties: { - engine: 'csv', + engine, source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS, source_prefix, - file_format_params: { sep, encoding, header: Boolean(header) }, - is_a_upload_csv: true, + file_format_params, }, input_id: undefined, };