diff --git a/docsfera.json b/docsfera.json index 00dea83..f509fa0 100644 --- a/docsfera.json +++ b/docsfera.json @@ -2444,16 +2444,71 @@ ] } }, - "/pipelinesV2/upload": { + "/pipelinesV2/init-upload": { "post": { - "operationId": "PipelinesController_uploadFile", + "operationId": "PipelinesController_initUploadFile", "parameters": [], "requestBody": { "required": true, "content": { - "multipart/form-data": { + "application/json": { "schema": { - "$ref": "#/components/schemas/IUploadCSVFile" + "$ref": "#/components/schemas/IInitUploadCSVFile" + } + } + } + }, + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "PipelinesV2" + ] + } + }, + "/pipelinesV2/complete-upload": { + "post": { + "operationId": "PipelinesController_completeUploadFile", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ICompleteUploadCSVFile" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ] + } + }, + "/pipelinesV2/file": { + "post": { + "operationId": "PipelinesController_uploadedFile", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ICreatePipelineCSVFile" } } } @@ -3350,7 +3405,7 @@ } }, "info": { - "title": "Maestro - fix/can-untoggle", + "title": "Maestro - feature/upload-to-s3-presigned", "description": "Documentation for Maestro gateway", "version": "1.0.0", "contact": {} @@ -5246,16 +5301,52 @@ "updated_at" ] }, - "IUploadCSVFile": { + "IInitUploadCSVFile": { "type": "object", "properties": { - "file": { - "type": "string", - "format": "binary" + "file_name": { + "type": "string" }, + "parts": { + "type": "number" + } + }, + "required": [ + "file_name", + "parts" + ] + }, + "ICompleteUploadCSVFile": { + "type": "object", + "properties": { + "upload_id": { + "type": "string" + }, + "file_name": { + "type": "string" + }, + "parts": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "upload_id", + "file_name", + "parts" + ] + }, + "ICreatePipelineCSVFile": { + "type": "object", + "properties": { "name": { "type": "string" }, + "file_name": { + "type": "string" + }, "description": { "type": "string" }, @@ -5270,8 +5361,8 @@ } }, "required": [ - "file", "name", + "file_name", "description", "encoding", "sep", diff --git a/package-lock.json b/package-lock.json index ba14a14..49a0c80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.27.2", + "@dadosfera/protospack-v2": "3.28.0", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -1727,9 +1727,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.27.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz", - "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==", + "version": "3.28.0", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz", + "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==", "dependencies": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", @@ -12284,9 +12284,9 @@ } }, "@dadosfera/protospack-v2": { - "version": "3.27.2", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz", - "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==", + "version": "3.28.0", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz", + "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==", "requires": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", diff --git a/package.json b/package.json index b0a77da..6525a77 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.27.2", + "@dadosfera/protospack-v2": "3.28.0", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 2759545..9cdd9dd 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -67,13 +67,32 @@ export interface IGetPipelineLogsRequest { details: string; } -export class IUploadCSVFile { - @ApiProperty({ format: 'binary' }) - file: string; +export class IInitUploadCSVFile { + @ApiProperty() + file_name: string; + @ApiProperty() + parts: number; +} + +export class ICompleteUploadCSVFile { + @ApiProperty() + upload_id: string; + + @ApiProperty() + file_name: string; + + @ApiProperty() + parts: { ETag: string; PartNumber: number }[]; +} + +export class ICreatePipelineCSVFile { @ApiProperty() name: string; + @ApiProperty() + file_name: string; + @ApiProperty() description: string; diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index c501d40..7552cb4 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -13,12 +13,9 @@ import { HttpCode, HttpStatus, Patch, - UseInterceptors, - UploadedFile, HttpException, } from '@nestjs/common'; import { - ApiConsumes, ApiCreatedResponse, ApiNoContentResponse, ApiTags, @@ -36,12 +33,13 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service'; import { + ICompleteUploadCSVFile, + ICreatePipelineCSVFile, ICreatePipelineV2Req, IPipelineV2, - IUploadCSVFile, + IInitUploadCSVFile, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; -import { FileInterceptor } from '@nestjs/platform-express'; @UseFilters(new GrpcToHttpExceptionFilter()) @ApiTags('PipelinesV2') @@ -300,37 +298,27 @@ export class PipelinesController { this.logger.info('PipelinesController - delete: OK'); } - @Post('/upload') + @Post('/init-upload') @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) - @ApiConsumes('multipart/form-data') - @UseInterceptors( - FileInterceptor('file', { limits: { fileSize: 50 * 1000 * 1000 + 1 } }), - ) - async uploadFile( + async initUploadFile( @User() user: RequestUser, - @UploadedFile() file, - @Body() body: IUploadCSVFile, + @Body() body: IInitUploadCSVFile, ) { - this.logger.info('/upload - Upload Connector Route'); + this.logger.info('/upload - Init Upload File Route'); const metadata = PackTheMetadata({ ...user }); - const parts_of_file_name = file.originalname.split('.'); - const file_format = parts_of_file_name.pop(); - const file_name = parts_of_file_name.join('.'); - let name = `${new Date().getTime()}_${file_name}`; - body.name ? (name = `${new Date().getTime()}_${body.name}`) : name; + const { file_name, parts } = body; - const source_prefix = `${user.customer_name}/${name}.${file_format}`; + const { urls, upload_id } = + await this.pipelinesClientService.initUploadFile( + { + name: file_name, + parts: String(parts), + }, + metadata, + ); - const response = await this.pipelinesClientService.uploadFile( - { - file, - name, - }, - metadata, - ); - - if (!response.url) { + if (!urls) { this.logger.info('pipeline/upload - Failed File pipeline'); throw new HttpException( 'Upload failed, try again in a few minutes, if the problem persists, contact support.', @@ -338,7 +326,37 @@ export class PipelinesController { ); } - const { sep, header, encoding, description } = body; + return { urls: JSON.parse(urls), upload_id }; + } + + @Post('/complete-upload') + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) + async completeUploadFile( + @User() user: RequestUser, + @Body() body: ICompleteUploadCSVFile, + ) { + this.logger.info('/upload - Complete Upload File Route'); + const metadata = PackTheMetadata({ ...user }); + + const { upload_id, parts, file_name } = body; + + return await this.pipelinesClientService.completeUploadFile( + { upload_id, parts, file_name }, + metadata, + ); + } + + @Post('/file') + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE) + async uploadedFile( + @User() user: RequestUser, + @Body() body: ICreatePipelineCSVFile, + ) { + this.logger.info('/upload - Upload Connector Route'); + const metadata = PackTheMetadata({ ...user }); + + const { name, sep, header, encoding, description, file_name } = body; + const source_prefix = `${user.customer_name}/${file_name}`; const upload_pipeline = { connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION, @@ -346,7 +364,7 @@ export class PipelinesController { connector_plugin: 'aws_s3', connector_version: '1.0.0', image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg', - name: body.name || file_name, + name, description, transformations_ids: [], tags: [], diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 1b888b0..de62496 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -261,13 +261,10 @@ export class PipelinesService implements OnModuleInit { return { objects: JSON.parse(objects) }; } - async uploadFile(uploadFile, metadata) { + async initUploadFile(uploadFile, metadata) { const body: Messages.PipelineV2UploadFileRequest = { - file: { - buffer: uploadFile.file.buffer, - mimetypes: uploadFile.file.mimetype, - }, name: uploadFile.name, + parts: uploadFile.parts, }; return lastValueFrom( @@ -293,4 +290,21 @@ export class PipelinesService implements OnModuleInit { return url; } + + async completeUploadFile(uploadFile, metadata) { + const body: Messages.PipelineV2CompleteUploadFileRequest = { + upload_id: uploadFile.upload_id, + parts: JSON.stringify(uploadFile.parts), + name: uploadFile.file_name, + }; + + return lastValueFrom( + this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata), + ).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + } }