From 5419f3690c18ab5e4ad21a615db77b014021a630 Mon Sep 17 00:00:00 2001 From: Anderson Feitosa Date: Mon, 16 Jan 2023 10:55:27 -0300 Subject: [PATCH 1/5] FEAT: update endpoint to get signed url to upload csv file --- docsfera.json | 48 ++++++++++++++++--- src/modules/pipelinesV2/interfaces.ts | 9 +++- .../pipelinesV2/pipelines.controller.ts | 46 ++++++++---------- src/modules/pipelinesV2/pipelines.service.ts | 4 -- 4 files changed, 69 insertions(+), 38 deletions(-) diff --git a/docsfera.json b/docsfera.json index 00dea83..0b2b1ce 100644 --- a/docsfera.json +++ b/docsfera.json @@ -2451,13 +2451,37 @@ "requestBody": { "required": true, "content": { - "multipart/form-data": { + "application/json": { "schema": { "$ref": "#/components/schemas/IUploadCSVFile" } } } }, + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "PipelinesV2" + ] + } + }, + "/pipelinesV2/file": { + "post": { + "operationId": "PipelinesController_uploadedFile", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ICreatePipelineCSVFile" + } + } + } + }, "responses": { "201": { "description": "", @@ -3350,7 +3374,7 @@ } }, "info": { - "title": "Maestro - fix/can-untoggle", + "title": "Maestro - feature/upload-to-s3-presigned", "description": "Documentation for Maestro gateway", "version": "1.0.0", "contact": {} @@ -5249,13 +5273,23 @@ "IUploadCSVFile": { "type": "object", "properties": { - "file": { - "type": "string", - "format": "binary" - }, + "file_name": { + "type": "string" + } + }, + "required": [ + "file_name" + ] + }, + "ICreatePipelineCSVFile": { + "type": "object", + "properties": { "name": { "type": "string" }, + "file_name": { + "type": "string" + }, "description": { "type": "string" }, @@ -5270,8 +5304,8 @@ } }, "required": [ - "file", "name", + "file_name", "description", "encoding", "sep", diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 2759545..66bc81c 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -68,12 +68,17 @@ export interface IGetPipelineLogsRequest { } export class IUploadCSVFile { - @ApiProperty({ format: 'binary' }) - file: string; + @ApiProperty() + file_name: string; +} +export class ICreatePipelineCSVFile { @ApiProperty() name: string; + @ApiProperty() + file_name: string; + @ApiProperty() description: string; diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index c501d40..8db88b8 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -13,12 +13,9 @@ import { HttpCode, HttpStatus, Patch, - UseInterceptors, - UploadedFile, HttpException, } from '@nestjs/common'; import { - ApiConsumes, ApiCreatedResponse, ApiNoContentResponse, ApiTags, @@ -36,12 +33,12 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { + ICreatePipelineCSVFile, ICreatePipelineV2Req, IPipelineV2, IUploadCSVFile, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; -import { FileInterceptor } from '@nestjs/platform-express'; @UseFilters(new GrpcToHttpExceptionFilter()) @ApiTags('PipelinesV2') @@ -302,35 +299,20 @@ export class PipelinesController { @Post('/upload') @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) - @ApiConsumes('multipart/form-data') - @UseInterceptors( - FileInterceptor('file', { limits: { fileSize: 50 * 1000 * 1000 + 1 } }), - ) - async uploadFile( - @User() user: RequestUser, - @UploadedFile() file, - @Body() body: IUploadCSVFile, - ) { + async uploadFile(@User() user: RequestUser, @Body() body: IUploadCSVFile) { this.logger.info('/upload - Upload Connector Route'); const metadata = PackTheMetadata({ ...user }); - const parts_of_file_name = file.originalname.split('.'); - const file_format = parts_of_file_name.pop(); - const file_name = parts_of_file_name.join('.'); - let name = `${new Date().getTime()}_${file_name}`; - body.name ? (name = `${new Date().getTime()}_${body.name}`) : name; + const { file_name } = body; - const source_prefix = `${user.customer_name}/${name}.${file_format}`; - - const response = await this.pipelinesClientService.uploadFile( + const { url } = await this.pipelinesClientService.uploadFile( { - file, - name, + name: file_name, }, metadata, ); - if (!response.url) { + if (!url) { this.logger.info('pipeline/upload - Failed File pipeline'); throw new HttpException( 'Upload failed, try again in a few minutes, if the problem persists, contact support.', @@ -338,6 +320,20 @@ export class PipelinesController { ); } + return { url }; + } + + @Post('/file') + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) + async uploadedFile( + @User() user: RequestUser, + @Body() body: ICreatePipelineCSVFile, + ) { + this.logger.info('/upload - Upload Connector Route'); + const metadata = PackTheMetadata({ ...user }); + + const source_prefix = `${user.customer_name}/${body.file_name}`; + const { sep, header, encoding, description } = body; const upload_pipeline = { @@ -346,7 +342,7 @@ export class PipelinesController { connector_plugin: 'aws_s3', connector_version: '1.0.0', image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg', - name: body.name || file_name, + name: body.name, description, transformations_ids: [], tags: [], diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 1b888b0..7de8a75 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -263,10 +263,6 @@ export class PipelinesService implements OnModuleInit { async uploadFile(uploadFile, metadata) { const body: Messages.PipelineV2UploadFileRequest = { - file: { - buffer: uploadFile.file.buffer, - mimetypes: uploadFile.file.mimetype, - }, name: uploadFile.name, }; From 131b166d8da1b7168693558e60015780c7056c4f Mon Sep 17 00:00:00 2001 From: Anderson Feitosa Date: Mon, 16 Jan 2023 10:56:03 -0300 Subject: [PATCH 2/5] REFACTOR: get data from body --- src/modules/pipelinesV2/pipelines.controller.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 8db88b8..937e371 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -332,9 +332,8 @@ export class PipelinesController { this.logger.info('/upload - Upload Connector Route'); const metadata = PackTheMetadata({ ...user }); - const source_prefix = `${user.customer_name}/${body.file_name}`; - - const { sep, header, encoding, description } = body; + const { name, sep, header, encoding, description, file_name } = body; + const source_prefix = `${user.customer_name}/${file_name}`; const upload_pipeline = { connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION, @@ -342,7 +341,7 @@ export class PipelinesController { connector_plugin: 'aws_s3', connector_version: '1.0.0', image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg', - name: body.name, + name, description, transformations_ids: [], tags: [], From 962e094a65bf52b61873aad755c659c633d03ff6 Mon Sep 17 00:00:00 2001 From: Anderson Feitosa Date: Thu, 19 Jan 2023 17:17:33 -0300 Subject: [PATCH 3/5] FEAT: add upload init multi part upload --- docsfera.json | 67 +++++++++++++++++-- package-lock.json | 14 ++-- package.json | 2 +- src/modules/pipelinesV2/interfaces.ts | 16 ++++- .../pipelinesV2/pipelines.controller.ts | 49 ++++++++++---- src/modules/pipelinesV2/pipelines.service.ts | 20 +++++- 6 files changed, 140 insertions(+), 28 deletions(-) diff --git a/docsfera.json b/docsfera.json index 0b2b1ce..f509fa0 100644 --- a/docsfera.json +++ b/docsfera.json @@ -2444,16 +2444,16 @@ ] } }, - "/pipelinesV2/upload": { + "/pipelinesV2/init-upload": { "post": { - "operationId": "PipelinesController_uploadFile", + "operationId": "PipelinesController_initUploadFile", "parameters": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IUploadCSVFile" + "$ref": "#/components/schemas/IInitUploadCSVFile" } } } @@ -2468,6 +2468,37 @@ ] } }, + "/pipelinesV2/complete-upload": { + "post": { + "operationId": "PipelinesController_completeUploadFile", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ICompleteUploadCSVFile" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ] + } + }, "/pipelinesV2/file": { "post": { "operationId": "PipelinesController_uploadedFile", @@ -5270,15 +5301,41 @@ "updated_at" ] }, - "IUploadCSVFile": { + "IInitUploadCSVFile": { "type": "object", "properties": { "file_name": { "type": "string" + }, + "parts": { + "type": "number" } }, "required": [ - "file_name" + "file_name", + "parts" + ] + }, + "ICompleteUploadCSVFile": { + "type": "object", + "properties": { + "upload_id": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "parts": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "upload_id", + "file_name", + "parts" ] }, "ICreatePipelineCSVFile": { diff --git a/package-lock.json b/package-lock.json index ba14a14..9b65783 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.27.2", + "@dadosfera/protospack-v2": "^3.28.0-beta.5", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -1727,9 +1727,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.27.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz", - "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==", + "version": "3.28.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz", + "integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==", "dependencies": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", @@ -12284,9 +12284,9 @@ } }, "@dadosfera/protospack-v2": { - "version": "3.27.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz", - "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==", + "version": "3.28.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz", + "integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==", "requires": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", diff --git a/package.json b/package.json index b0a77da..d20c6c4 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.27.2", + "@dadosfera/protospack-v2": "^3.28.0-beta.5", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 66bc81c..9cdd9dd 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -67,9 +67,23 @@ export interface IGetPipelineLogsRequest { details: string; } -export class IUploadCSVFile { +export class IInitUploadCSVFile { @ApiProperty() file_name: string; + + @ApiProperty() + parts: number; +} + +export class ICompleteUploadCSVFile { + @ApiProperty() + upload_id: string; + + @ApiProperty() + file_name: string; + + @ApiProperty() + parts: { ETag: string; PartNumber: number }[]; } export class ICreatePipelineCSVFile { diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 937e371..7552cb4 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -33,10 +33,11 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { + ICompleteUploadCSVFile, ICreatePipelineCSVFile, ICreatePipelineV2Req, IPipelineV2, - IUploadCSVFile, + IInitUploadCSVFile, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @@ -297,22 +298,27 @@ export class PipelinesController { this.logger.info('PipelinesController - delete: OK'); } - @Post('/upload') + @Post('/init-upload') @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) - async uploadFile(@User() user: RequestUser, @Body() body: IUploadCSVFile) { - this.logger.info('/upload - Upload Connector Route'); + async initUploadFile( + @User() user: RequestUser, + @Body() body: IInitUploadCSVFile, + ) { + this.logger.info('/upload - Init Upload File Route'); const metadata = PackTheMetadata({ ...user }); - const { file_name } = body; + const { file_name, parts } = body; - const { url } = await this.pipelinesClientService.uploadFile( - { - name: file_name, - }, - metadata, - ); + const { urls, upload_id } = + await this.pipelinesClientService.initUploadFile( + { + name: file_name, + parts: String(parts), + }, + metadata, + ); - if (!url) { + if (!urls) { this.logger.info('pipeline/upload - Failed File pipeline'); throw new HttpException( 'Upload failed, try again in a few minutes, if the problem persists, contact support.', @@ -320,7 +326,24 @@ export class PipelinesController { ); } - return { url }; + return { urls: JSON.parse(urls), upload_id }; + } + + @Post('/complete-upload') + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) + async completeUploadFile( + @User() user: RequestUser, + @Body() body: ICompleteUploadCSVFile, + ) { + this.logger.info('/upload - Complete Upload File Route'); + const metadata = PackTheMetadata({ ...user }); + + const { upload_id, parts, file_name } = body; + + return await this.pipelinesClientService.completeUploadFile( + { upload_id, parts, file_name }, + metadata, + ); } @Post('/file') diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 7de8a75..25145e9 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -261,9 +261,10 @@ export class PipelinesService implements OnModuleInit { return { objects: JSON.parse(objects) }; } - async uploadFile(uploadFile, metadata) { + async initUploadFile(uploadFile, metadata) { const body: Messages.PipelineV2UploadFileRequest = { name: uploadFile.name, + parts: uploadFile.parts, }; return lastValueFrom( @@ -289,4 +290,21 @@ export class PipelinesService implements OnModuleInit { return url; } + + async completeUploadFile(uploadFile, metadata) { + const body: Messages.PipelineV2CompleteUploadFileRequest = { + upload_id: uploadFile.name, + parts: uploadFile.parts, + name: uploadFile.file_name, + }; + + return lastValueFrom( + this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata), + ).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + } } From 5c24ec5560d6f9b71d4b619743421981b88e099f Mon Sep 17 00:00:00 2001 From: Anderson Feitosa Date: Fri, 20 Jan 2023 10:05:36 -0300 Subject: [PATCH 4/5] FEAT: fix upload id --- src/modules/pipelinesV2/pipelines.service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 25145e9..de62496 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -293,8 +293,8 @@ export class PipelinesService implements OnModuleInit { async completeUploadFile(uploadFile, metadata) { const body: Messages.PipelineV2CompleteUploadFileRequest = { - upload_id: uploadFile.name, - parts: uploadFile.parts, + upload_id: uploadFile.upload_id, + parts: JSON.stringify(uploadFile.parts), name: uploadFile.file_name, }; From 1ef9fd429832ba08168e3ec7eca9d74b2f58e6c5 Mon Sep 17 00:00:00 2001 From: Anderson Feitosa Date: Mon, 23 Jan 2023 13:56:51 -0300 Subject: [PATCH 5/5] FEAT: update packages --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9b65783..49a0c80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.28.0-beta.5", + "@dadosfera/protospack-v2": "3.28.0", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -1727,9 +1727,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.28.0-beta.5", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz", - "integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==", + "version": "3.28.0", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz", + "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==", "dependencies": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", @@ -12284,9 +12284,9 @@ } }, "@dadosfera/protospack-v2": { - "version": "3.28.0-beta.5", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz", - "integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==", + "version": "3.28.0", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz", + "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==", "requires": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", diff --git a/package.json b/package.json index d20c6c4..6525a77 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "^3.28.0-beta.5", + "@dadosfera/protospack-v2": "3.28.0", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7",