gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-11-07 21:59:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

FEAT: add upload init multi part upload

Browse Source
This commit is contained in:
Anderson Feitosa
2023-01-19 17:17:33 -03:00
parent 131b166d8d
commit 962e094a65
6 changed files with 140 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docsfera.json
+62 -5
View File
@@ -2444,16 +2444,16 @@
]
}
},
"/pipelinesV2/upload": {
"/pipelinesV2/init-upload": {
"post": {
"operationId": "PipelinesController_uploadFile",
"operationId": "PipelinesController_initUploadFile",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/IUploadCSVFile"
"$ref": "#/components/schemas/IInitUploadCSVFile"
}
}
}
@@ -2468,6 +2468,37 @@
]
}
},
"/pipelinesV2/complete-upload": {
"post": {
"operationId": "PipelinesController_completeUploadFile",
"parameters": [],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ICompleteUploadCSVFile"
}
}
}
},
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
]
}
},
"/pipelinesV2/file": {
"post": {
"operationId": "PipelinesController_uploadedFile",
@@ -5270,15 +5301,41 @@
"updated_at"
]
},
"IUploadCSVFile": {
"IInitUploadCSVFile": {
"type": "object",
"properties": {
"file_name": {
"type": "string"
},
"parts": {
"type": "number"
}
},
"required": [
"file_name"
"file_name",
"parts"
]
},
"ICompleteUploadCSVFile": {
"type": "object",
"properties": {
"upload_id": {
"type": "string"
},
"file_name": {
"type": "string"
},
"parts": {
"type": "array",
"items": {
"type": "string"
}
}
},
"required": [
"upload_id",
"file_name",
"parts"
]
},
"ICreatePipelineCSVFile": {
package-lock.json
Generated
+7 -7
View File
@@ -12,7 +12,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.27.2",
"@dadosfera/protospack-v2": "^3.28.0-beta.5",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.27.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz",
"integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==",
"version": "3.28.0-beta.5",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz",
"integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==",
"dependencies": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -12284,9 +12284,9 @@
}
},
"@dadosfera/protospack-v2": {
"version": "3.27.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz",
"integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==",
"version": "3.28.0-beta.5",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0-beta.5.tgz",
"integrity": "sha512-QDqylnH2hZZhybDIXx0ZbLoteqADeFMgtbWvzQzwzRpxBhVPcS9OU6NRaF9vEpR8YXXAknoy1IyHCkOQ/MSkIQ==",
"requires": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
package.json
+1 -1
View File
@@ -28,7 +28,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.27.2",
"@dadosfera/protospack-v2": "^3.28.0-beta.5",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
src/modules/pipelinesV2/interfaces.ts
+15 -1
View File
@@ -67,9 +67,23 @@ export interface IGetPipelineLogsRequest {
details: string;
}
export class IUploadCSVFile {
export class IInitUploadCSVFile {
@ApiProperty()
file_name: string;
@ApiProperty()
parts: number;
}
export class ICompleteUploadCSVFile {
@ApiProperty()
upload_id: string;
@ApiProperty()
file_name: string;
@ApiProperty()
parts: { ETag: string; PartNumber: number }[];
}
export class ICreatePipelineCSVFile {
src/modules/pipelinesV2/pipelines.controller.ts
+36 -13
View File
@@ -33,10 +33,11 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
ICompleteUploadCSVFile,
ICreatePipelineCSVFile,
ICreatePipelineV2Req,
IPipelineV2,
IUploadCSVFile,
IInitUploadCSVFile,
} from './interfaces';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@@ -297,22 +298,27 @@ export class PipelinesController {
this.logger.info('PipelinesController - delete: OK');
}
@Post('/upload')
@Post('/init-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async uploadFile(@User() user: RequestUser, @Body() body: IUploadCSVFile) {
this.logger.info('/upload - Upload Connector Route');
async initUploadFile(
@User() user: RequestUser,
@Body() body: IInitUploadCSVFile,
) {
this.logger.info('/upload - Init Upload File Route');
const metadata = PackTheMetadata({ ...user });
const { file_name } = body;
const { file_name, parts } = body;
const { url } = await this.pipelinesClientService.uploadFile(
{
name: file_name,
},
metadata,
);
const { urls, upload_id } =
await this.pipelinesClientService.initUploadFile(
{
name: file_name,
parts: String(parts),
},
metadata,
);
if (!url) {
if (!urls) {
this.logger.info('pipeline/upload - Failed File pipeline');
throw new HttpException(
'Upload failed, try again in a few minutes, if the problem persists, contact support.',
@@ -320,7 +326,24 @@ export class PipelinesController {
);
}
return { url };
return { urls: JSON.parse(urls), upload_id };
}
@Post('/complete-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async completeUploadFile(
@User() user: RequestUser,
@Body() body: ICompleteUploadCSVFile,
) {
this.logger.info('/upload - Complete Upload File Route');
const metadata = PackTheMetadata({ ...user });
const { upload_id, parts, file_name } = body;
return await this.pipelinesClientService.completeUploadFile(
{ upload_id, parts, file_name },
metadata,
);
}
@Post('/file')
src/modules/pipelinesV2/pipelines.service.ts
+19 -1
View File
@@ -261,9 +261,10 @@ export class PipelinesService implements OnModuleInit {
return { objects: JSON.parse(objects) };
}
async uploadFile(uploadFile, metadata) {
async initUploadFile(uploadFile, metadata) {
const body: Messages.PipelineV2UploadFileRequest = {
name: uploadFile.name,
parts: uploadFile.parts,
};
return lastValueFrom(
@@ -289,4 +290,21 @@ export class PipelinesService implements OnModuleInit {
return url;
}
async completeUploadFile(uploadFile, metadata) {
const body: Messages.PipelineV2CompleteUploadFileRequest = {
upload_id: uploadFile.name,
parts: uploadFile.parts,
name: uploadFile.file_name,
};
return lastValueFrom(
this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
}
}