Merge pull request #189 from dadosfera/feat/upload-csv

FIX: Upload CSV

README.md

@@ -290,7 +290,7 @@ Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
 ### Making a Pull Request
 1. Commit your changes
 2. Open the Pull Request on GitHub
-3. Send Pull Request link in microsfera Google Chat Group for review and possible approval
+3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
 
 ## 🛠️ Built with
 Some technologies used in this project:

docsfera.json

@@ -264,6 +264,20 @@
           ]
         }
       },
+      "/auth/verify-access-token": {
+        "get": {
+          "operationId": "AuthController_verifyAccessToken",
+          "parameters": [],
+          "responses": {
+            "200": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Auth"
+          ]
+        }
+      },
       "/connectors": {
         "post": {
           "operationId": "ConnectorController_uploadConnector",
@@ -2381,6 +2395,37 @@
           ]
         }
       },
+      "/pipelinesV2/upload": {
+        "post": {
+          "operationId": "PipelinesController_uploadFile",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "multipart/form-data": {
+                "schema": {
+                  "$ref": "#/components/schemas/IUploadCSVFile"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
       "/inputs/available-entities/{plugin}": {
         "get": {
           "operationId": "InputsController_getAvailableEntities",
@@ -3242,7 +3287,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/embed-data-asset",
+      "title": "Maestro - feat/upload-csv",
       "description": "Documentation for Maestro gateway",
       "version": "1.0.0",
       "contact": {}
@@ -5107,6 +5152,38 @@
             "updated_at"
           ]
         },
+        "IUploadCSVFile": {
+          "type": "object",
+          "properties": {
+            "file": {
+              "type": "string",
+              "format": "binary"
+            },
+            "name": {
+              "type": "string"
+            },
+            "description": {
+              "type": "string"
+            },
+            "encoding": {
+              "type": "string"
+            },
+            "sep": {
+              "type": "string"
+            },
+            "header": {
+              "type": "boolean"
+            }
+          },
+          "required": [
+            "file",
+            "name",
+            "description",
+            "encoding",
+            "sep",
+            "header"
+          ]
+        },
         "AvailableEntity": {
           "type": "object",
           "properties": {

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.22.0",
+        "@dadosfera/protospack-v2": "3.23.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.22.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
-      "integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
+      "version": "3.23.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.23.0.tgz",
+      "integrity": "sha512-pe2s7aJYaNlUhQ+QoMGkJoFjTS8Dh+xSgUClSJuCsTia17oidRAucfpeMUX9ws1a3+tDrXTpH519CxeHzuZQPA==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12324,9 +12324,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.22.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
-      "integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
+      "version": "3.23.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.23.0.tgz",
+      "integrity": "sha512-pe2s7aJYaNlUhQ+QoMGkJoFjTS8Dh+xSgUClSJuCsTia17oidRAucfpeMUX9ws1a3+tDrXTpH519CxeHzuZQPA==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.22.0",
+    "@dadosfera/protospack-v2": "3.23.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/modules/auth/auth.controller.ts

@@ -7,6 +7,7 @@ import {
   HttpStatus,
   Inject,
   UseFilters,
+  Get,
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import {
@@ -21,7 +22,10 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/authentication/authentication.decorator';
 import { AuthClientService } from './auth.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
@@ -185,4 +189,11 @@ export class AuthController {
 
     return this.authClient.verifyTotp({ accessToken, totp });
   }
+
+  @Authenticated()
+  @Get('verify-access-token')
+  @HttpCode(HttpStatus.OK)
+  verifyAccessToken() {
+    return { access_token_status: 'valid' };
+  }
 }

src/modules/pipelinesV2/interfaces.ts

@@ -66,3 +66,23 @@ export interface IGetPipelineLogsRequest {
   id: string;
   details: string;
 }
+
+export class IUploadCSVFile {
+  @ApiProperty({ format: 'binary' })
+  file: string;
+
+  @ApiProperty()
+  name: string;
+
+  @ApiProperty()
+  description: string;
+
+  @ApiProperty()
+  encoding: string;
+
+  @ApiProperty()
+  sep: string;
+
+  @ApiProperty()
+  header: boolean;
+}

src/modules/pipelinesV2/pipelines.controller.ts

@@ -13,13 +13,20 @@ import {
   HttpCode,
   HttpStatus,
   Patch,
+  UseInterceptors,
+  UploadedFile,
+  HttpException,
 } from '@nestjs/common';
 import {
+  ApiConsumes,
   ApiCreatedResponse,
   ApiNoContentResponse,
   ApiTags,
 } from '@nestjs/swagger';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
+import {
+  AuthenticateCondition,
+  RequireAllPermissions,
+} from 'src/authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -28,8 +35,13 @@ import { RequestUser, User } from 'src/authentication/user.decorator';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
-import { ICreatePipelineV2Req, IPipelineV2 } from './interfaces';
+import {
+  ICreatePipelineV2Req,
+  IPipelineV2,
+  IUploadCSVFile,
+} from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+import { FileInterceptor } from '@nestjs/platform-express';
 
 @UseFilters(new GrpcToHttpExceptionFilter())
 @ApiTags('PipelinesV2')
@@ -240,4 +252,74 @@ export class PipelinesController {
     await this.pipelinesClientService.remove({ id, metadata, user });
     this.logger.info('PipelinesController - delete: OK');
   }
+
+  @Post('/upload')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(
+    FileInterceptor('file', { limits: { fileSize: 50 * 1000 * 1000 + 1 } }),
+  )
+  async uploadFile(
+    @User() user: RequestUser,
+    @UploadedFile() file,
+    @Body() body: IUploadCSVFile,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const parts_of_file_name = file.originalname.split('.');
+    const file_format = parts_of_file_name.pop();
+    const file_name = parts_of_file_name.join('.');
+    let name = `${new Date().getTime()}_${file_name}`;
+    body.name ? (name = `${new Date().getTime()}_${body.name}`) : name;
+
+    const source_prefix = `${user.customer_name}/${name}.${file_format}`;
+
+    const response = await this.pipelinesClientService.uploadFile(
+      {
+        file,
+        name,
+      },
+      metadata,
+    );
+
+    if (!response.url) {
+      this.logger.info('pipeline/upload - Failed File pipeline');
+      throw new HttpException(
+        'Upload failed, try again in a few minutes, if the problem persists, contact support.',
+        HttpStatus.INTERNAL_SERVER_ERROR,
+      );
+    }
+
+    const { sep, header, encoding, description } = body;
+
+    const upload_pipeline = {
+      connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
+      connector_name: 'Amazon S3',
+      connector_plugin: 'aws_s3',
+      connector_version: '1.0.0',
+      image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
+      name: body.name || file_name,
+      description,
+      transformations_ids: [],
+      tags: [],
+      cron: '@once',
+      config: { cron: '@once', tables: [] },
+      properties: {
+        engine: 'csv',
+        source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
+        source_prefix,
+        file_format_params: { sep, encoding, header: Boolean(header) },
+        is_a_upload_csv: true,
+      },
+      input_id: undefined,
+    };
+
+    const pipeline = await this.pipelinesClientService.create(
+      upload_pipeline,
+      metadata,
+    );
+
+    return pipeline;
+  }
 }

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,4 +1,10 @@
-import { BadRequestException, Inject, OnModuleInit } from '@nestjs/common';
+import {
+  BadRequestException,
+  HttpException,
+  HttpStatus,
+  Inject,
+  OnModuleInit,
+} from '@nestjs/common';
 import { ClientGrpc } from '@nestjs/microservices';
 import {
   Messages,
@@ -243,4 +249,23 @@ export class PipelinesService implements OnModuleInit {
       },
     };
   }
+
+  async uploadFile(uploadFile, metadata) {
+    const body: Messages.PipelineV2UploadFileRequest = {
+      file: {
+        buffer: uploadFile.file.buffer,
+        mimetypes: uploadFile.file.mimetype,
+      },
+      name: uploadFile.name,
+    };
+
+    return lastValueFrom(
+      this.pipelineWriteService.PipelineV2UploadFile(body, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
 }