Merge pull request #201 from dadosfera/fix/oauth-facebook

FIX: add all scopes to facebook oauth

docsfera.json

@@ -2444,16 +2444,71 @@
           ]
         }
       },
-      "/pipelinesV2/upload": {
+      "/pipelinesV2/init-upload": {
         "post": {
-          "operationId": "PipelinesController_uploadFile",
+          "operationId": "PipelinesController_initUploadFile",
           "parameters": [],
           "requestBody": {
             "required": true,
             "content": {
-              "multipart/form-data": {
+              "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/IUploadCSVFile"
+                  "$ref": "#/components/schemas/IInitUploadCSVFile"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
+      "/pipelinesV2/complete-upload": {
+        "post": {
+          "operationId": "PipelinesController_completeUploadFile",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ICompleteUploadCSVFile"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
+      "/pipelinesV2/file": {
+        "post": {
+          "operationId": "PipelinesController_uploadedFile",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ICreatePipelineCSVFile"
                 }
               }
             }
@@ -3350,7 +3405,7 @@
       }
     },
     "info": {
-      "title": "Maestro - fix/can-untoggle",
+      "title": "Maestro - feature/upload-to-s3-presigned",
       "description": "Documentation for Maestro gateway",
       "version": "1.0.0",
       "contact": {}
@@ -5246,16 +5301,52 @@
             "updated_at"
           ]
         },
-        "IUploadCSVFile": {
+        "IInitUploadCSVFile": {
           "type": "object",
           "properties": {
-            "file": {
-              "type": "string",
-              "format": "binary"
+            "file_name": {
+              "type": "string"
             },
+            "parts": {
+              "type": "number"
+            }
+          },
+          "required": [
+            "file_name",
+            "parts"
+          ]
+        },
+        "ICompleteUploadCSVFile": {
+          "type": "object",
+          "properties": {
+            "upload_id": {
+              "type": "string"
+            },
+            "file_name": {
+              "type": "string"
+            },
+            "parts": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "upload_id",
+            "file_name",
+            "parts"
+          ]
+        },
+        "ICreatePipelineCSVFile": {
+          "type": "object",
+          "properties": {
             "name": {
               "type": "string"
             },
+            "file_name": {
+              "type": "string"
+            },
             "description": {
               "type": "string"
             },
@@ -5270,8 +5361,8 @@
             }
           },
           "required": [
-            "file",
             "name",
+            "file_name",
             "description",
             "encoding",
             "sep",

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.27.2",
+        "@dadosfera/protospack-v2": "3.28.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.27.2",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz",
-      "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==",
+      "version": "3.28.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz",
+      "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12284,9 +12284,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.27.2",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.2.tgz",
-      "integrity": "sha512-WDdneA/+AuRMrgbqU84NgaTw9HLp7HYWbVMEqxTEli+JizoiSYpAQ9ZmG6TnXgsTAdjZf7DOOVdY3OKNxRjWig==",
+      "version": "3.28.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz",
+      "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.27.2",
+    "@dadosfera/protospack-v2": "3.28.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/modules/oauth/passport-strategies/facebook-strategy.ts

@@ -44,7 +44,8 @@ export class FacebookStrategy extends PassportStrategy(Strategy) {
       req,
       'application',
     );
-    options.scope = 'ads_read ads_management';
+    options.scope =
+      'pages_show_list ads_read pages_read_engagement ads_management';
     super.authenticate(req, options);
   }
 }

src/modules/pipelinesV2/interfaces.ts

@@ -67,13 +67,32 @@ export interface IGetPipelineLogsRequest {
   details: string;
 }
 
-export class IUploadCSVFile {
-  @ApiProperty({ format: 'binary' })
-  file: string;
+export class IInitUploadCSVFile {
+  @ApiProperty()
+  file_name: string;
 
+  @ApiProperty()
+  parts: number;
+}
+
+export class ICompleteUploadCSVFile {
+  @ApiProperty()
+  upload_id: string;
+
+  @ApiProperty()
+  file_name: string;
+
+  @ApiProperty()
+  parts: { ETag: string; PartNumber: number }[];
+}
+
+export class ICreatePipelineCSVFile {
   @ApiProperty()
   name: string;
 
+  @ApiProperty()
+  file_name: string;
+
   @ApiProperty()
   description: string;
 

src/modules/pipelinesV2/pipelines.controller.ts

@@ -13,12 +13,9 @@ import {
   HttpCode,
   HttpStatus,
   Patch,
-  UseInterceptors,
-  UploadedFile,
   HttpException,
 } from '@nestjs/common';
 import {
-  ApiConsumes,
   ApiCreatedResponse,
   ApiNoContentResponse,
   ApiTags,
@@ -36,12 +33,13 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
 import {
+  ICompleteUploadCSVFile,
+  ICreatePipelineCSVFile,
   ICreatePipelineV2Req,
   IPipelineV2,
-  IUploadCSVFile,
+  IInitUploadCSVFile,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
-import { FileInterceptor } from '@nestjs/platform-express';
 
 @UseFilters(new GrpcToHttpExceptionFilter())
 @ApiTags('PipelinesV2')
@@ -300,37 +298,27 @@ export class PipelinesController {
     this.logger.info('PipelinesController - delete: OK');
   }
 
-  @Post('/upload')
+  @Post('/init-upload')
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
-  @ApiConsumes('multipart/form-data')
-  @UseInterceptors(
-    FileInterceptor('file', { limits: { fileSize: 50 * 1000 * 1000 + 1 } }),
-  )
-  async uploadFile(
+  async initUploadFile(
     @User() user: RequestUser,
-    @UploadedFile() file,
-    @Body() body: IUploadCSVFile,
+    @Body() body: IInitUploadCSVFile,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('/upload - Init Upload File Route');
     const metadata = PackTheMetadata({ ...user });
 
-    const parts_of_file_name = file.originalname.split('.');
-    const file_format = parts_of_file_name.pop();
-    const file_name = parts_of_file_name.join('.');
-    let name = `${new Date().getTime()}_${file_name}`;
-    body.name ? (name = `${new Date().getTime()}_${body.name}`) : name;
+    const { file_name, parts } = body;
 
-    const source_prefix = `${user.customer_name}/${name}.${file_format}`;
+    const { urls, upload_id } =
+      await this.pipelinesClientService.initUploadFile(
+        {
+          name: file_name,
+          parts: String(parts),
+        },
+        metadata,
+      );
 
-    const response = await this.pipelinesClientService.uploadFile(
-      {
-        file,
-        name,
-      },
-      metadata,
-    );
-
-    if (!response.url) {
+    if (!urls) {
       this.logger.info('pipeline/upload - Failed File pipeline');
       throw new HttpException(
         'Upload failed, try again in a few minutes, if the problem persists, contact support.',
@@ -338,7 +326,37 @@ export class PipelinesController {
       );
     }
 
-    const { sep, header, encoding, description } = body;
+    return { urls: JSON.parse(urls), upload_id };
+  }
+
+  @Post('/complete-upload')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  async completeUploadFile(
+    @User() user: RequestUser,
+    @Body() body: ICompleteUploadCSVFile,
+  ) {
+    this.logger.info('/upload - Complete Upload File Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const { upload_id, parts, file_name } = body;
+
+    return await this.pipelinesClientService.completeUploadFile(
+      { upload_id, parts, file_name },
+      metadata,
+    );
+  }
+
+  @Post('/file')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  async uploadedFile(
+    @User() user: RequestUser,
+    @Body() body: ICreatePipelineCSVFile,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const { name, sep, header, encoding, description, file_name } = body;
+    const source_prefix = `${user.customer_name}/${file_name}`;
 
     const upload_pipeline = {
       connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
@@ -346,7 +364,7 @@ export class PipelinesController {
       connector_plugin: 'aws_s3',
       connector_version: '1.0.0',
       image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
-      name: body.name || file_name,
+      name,
       description,
       transformations_ids: [],
       tags: [],

src/modules/pipelinesV2/pipelines.service.ts

@@ -261,13 +261,10 @@ export class PipelinesService implements OnModuleInit {
     return { objects: JSON.parse(objects) };
   }
 
-  async uploadFile(uploadFile, metadata) {
+  async initUploadFile(uploadFile, metadata) {
     const body: Messages.PipelineV2UploadFileRequest = {
-      file: {
-        buffer: uploadFile.file.buffer,
-        mimetypes: uploadFile.file.mimetype,
-      },
       name: uploadFile.name,
+      parts: uploadFile.parts,
     };
 
     return lastValueFrom(
@@ -293,4 +290,21 @@ export class PipelinesService implements OnModuleInit {
 
     return url;
   }
+
+  async completeUploadFile(uploadFile, metadata) {
+    const body: Messages.PipelineV2CompleteUploadFileRequest = {
+      upload_id: uploadFile.upload_id,
+      parts: JSON.stringify(uploadFile.parts),
+      name: uploadFile.file_name,
+    };
+
+    return lastValueFrom(
+      this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
 }