From 962e094a65bf52b61873aad755c659c633d03ff6 Mon Sep 17 00:00:00 2001 From: Anderson Feitosa Date: Thu, 19 Jan 2023 17:17:33 -0300 Subject: [PATCH] FEAT: add upload init multi part upload --- docsfera.json | 67 +++++++++++++++++-- package-lock.json | 14 ++-- package.json | 2 +- src/modules/pipelinesV2/interfaces.ts | 16 ++++- .../pipelinesV2/pipelines.controller.ts | 49 ++++++++++---- src/modules/pipelinesV2/pipelines.service.ts | 20 +++++- 6 files changed, 140 insertions(+), 28 deletions(-) diff --git a/docsfera.json b/docsfera.json index 0b2b1ce..f509fa0 100644 --- a/docsfera.json +++ b/docsfera.json @@ -2444,16 +2444,16 @@ ] } }, - "/pipelinesV2/upload": { + "/pipelinesV2/init-upload": { "post": { - "operationId": "PipelinesController_uploadFile", + "operationId": "PipelinesController_initUploadFile", "parameters": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IUploadCSVFile" + "$ref": "#/components/schemas/IInitUploadCSVFile" } } } @@ -2468,6 +2468,37 @@ ] } }, + "/pipelinesV2/complete-upload": { + "post": { + "operationId": "PipelinesController_completeUploadFile", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ICompleteUploadCSVFile" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ] + } + }, "/pipelinesV2/file": { "post": { "operationId": "PipelinesController_uploadedFile", @@ -5270,15 +5301,41 @@ "updated_at" ] }, - "IUploadCSVFile": { + "IInitUploadCSVFile": { "type": "object", "properties": { "file_name": { "type": "string" + }, + "parts": { + "type": "number" } }, "required": [ - "file_name" + "file_name", + "parts" + ] + }, + "ICompleteUploadCSVFile": { + "type": "object", + "properties": { + "upload_id": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "parts": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "upload_id", + "file_name", + "parts" ] }, "ICreatePipelineCSVFile": { diff --git a/package-lock.json b/package-lock.json index ba14a14..9b65783 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.27.2", + "@dadosfera/protospack-v2": "^3.28.0-beta.5", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -1727,9 +1727,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.27.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz", - "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==", + "version": "3.28.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz", + "integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==", "dependencies": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", @@ -12284,9 +12284,9 @@ } }, "@dadosfera/protospack-v2": { - "version": "3.27.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz", - "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==", + "version": "3.28.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz", + "integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==", "requires": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", diff --git a/package.json b/package.json index b0a77da..d20c6c4 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.27.2", + "@dadosfera/protospack-v2": "^3.28.0-beta.5", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 66bc81c..9cdd9dd 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -67,9 +67,23 @@ export interface IGetPipelineLogsRequest { details: string; } -export class IUploadCSVFile { +export class IInitUploadCSVFile { @ApiProperty() file_name: string; + + @ApiProperty() + parts: number; +} + +export class ICompleteUploadCSVFile { + @ApiProperty() + upload_id: string; + + @ApiProperty() + file_name: string; + + @ApiProperty() + parts: { ETag: string; PartNumber: number }[]; } export class ICreatePipelineCSVFile { diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 937e371..7552cb4 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -33,10 +33,11 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { + ICompleteUploadCSVFile, ICreatePipelineCSVFile, ICreatePipelineV2Req, IPipelineV2, - IUploadCSVFile, + IInitUploadCSVFile, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @@ -297,22 +298,27 @@ export class PipelinesController { this.logger.info('PipelinesController - delete: OK'); } - @Post('/upload') + @Post('/init-upload') @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) - async uploadFile(@User() user: RequestUser, @Body() body: IUploadCSVFile) { - this.logger.info('/upload - Upload Connector Route'); + async initUploadFile( + @User() user: RequestUser, + @Body() body: IInitUploadCSVFile, + ) { + this.logger.info('/upload - Init Upload File Route'); const metadata = PackTheMetadata({ ...user }); - const { file_name } = body; + const { file_name, parts } = body; - const { url } = await this.pipelinesClientService.uploadFile( - { - name: file_name, - }, - metadata, - ); + const { urls, upload_id } = + await this.pipelinesClientService.initUploadFile( + { + name: file_name, + parts: String(parts), + }, + metadata, + ); - if (!url) { + if (!urls) { this.logger.info('pipeline/upload - Failed File pipeline'); throw new HttpException( 'Upload failed, try again in a few minutes, if the problem persists, contact support.', @@ -320,7 +326,24 @@ export class PipelinesController { ); } - return { url }; + return { urls: JSON.parse(urls), upload_id }; + } + + @Post('/complete-upload') + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) + async completeUploadFile( + @User() user: RequestUser, + @Body() body: ICompleteUploadCSVFile, + ) { + this.logger.info('/upload - Complete Upload File Route'); + const metadata = PackTheMetadata({ ...user }); + + const { upload_id, parts, file_name } = body; + + return await this.pipelinesClientService.completeUploadFile( + { upload_id, parts, file_name }, + metadata, + ); } @Post('/file') diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 7de8a75..25145e9 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -261,9 +261,10 @@ export class PipelinesService implements OnModuleInit { return { objects: JSON.parse(objects) }; } - async uploadFile(uploadFile, metadata) { + async initUploadFile(uploadFile, metadata) { const body: Messages.PipelineV2UploadFileRequest = { name: uploadFile.name, + parts: uploadFile.parts, }; return lastValueFrom( @@ -289,4 +290,21 @@ export class PipelinesService implements OnModuleInit { return url; } + + async completeUploadFile(uploadFile, metadata) { + const body: Messages.PipelineV2CompleteUploadFileRequest = { + upload_id: uploadFile.name, + parts: uploadFile.parts, + name: uploadFile.file_name, + }; + + return lastValueFrom( + this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata), + ).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + } }