Merge pull request #200 from dadosfera/feature/upload-to-s3-presigned

feature/upload-to-s3-presigned

docsfera.json

@@ -1587,6 +1587,17 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "canUntoggleLogic",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "enum": [
+                  "data_assets_sharing"
+                ],
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -2433,16 +2444,71 @@
           ]
         }
       },
-      "/pipelinesV2/upload": {
+      "/pipelinesV2/init-upload": {
         "post": {
-          "operationId": "PipelinesController_uploadFile",
+          "operationId": "PipelinesController_initUploadFile",
           "parameters": [],
           "requestBody": {
             "required": true,
             "content": {
-              "multipart/form-data": {
+              "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/IUploadCSVFile"
+                  "$ref": "#/components/schemas/IInitUploadCSVFile"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
+      "/pipelinesV2/complete-upload": {
+        "post": {
+          "operationId": "PipelinesController_completeUploadFile",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ICompleteUploadCSVFile"
+                }
+              }
+            }
+          },
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "PipelinesV2"
+          ]
+        }
+      },
+      "/pipelinesV2/file": {
+        "post": {
+          "operationId": "PipelinesController_uploadedFile",
+          "parameters": [],
+          "requestBody": {
+            "required": true,
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ICreatePipelineCSVFile"
                 }
               }
             }
@@ -3339,7 +3405,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feature/download-logs",
+      "title": "Maestro - feature/upload-to-s3-presigned",
       "description": "Documentation for Maestro gateway",
       "version": "1.0.0",
       "contact": {}
@@ -3390,13 +3456,17 @@
             },
             "tier": {
               "type": "string"
+            },
+            "scheduleLimit": {
+              "type": "string"
             }
           },
           "required": [
             "modules",
             "id",
             "name",
-            "tier"
+            "tier",
+            "scheduleLimit"
           ]
         },
         "AuthUser": {
@@ -3989,6 +4059,10 @@
             },
             "usage": {
               "type": "string"
+            },
+            "canUntoggle": {
+              "type": "boolean",
+              "description": "Whether this role can be untoggled"
             }
           },
           "required": [
@@ -4466,6 +4540,10 @@
             },
             "usage": {
               "type": "string"
+            },
+            "canUntoggle": {
+              "type": "boolean",
+              "description": "Whether this role can be untoggled"
             }
           },
           "required": [
@@ -4556,6 +4634,10 @@
             },
             "usage": {
               "type": "string"
+            },
+            "canUntoggle": {
+              "type": "boolean",
+              "description": "Whether this role can be untoggled"
             }
           },
           "required": [
@@ -5219,16 +5301,52 @@
             "updated_at"
           ]
         },
-        "IUploadCSVFile": {
+        "IInitUploadCSVFile": {
           "type": "object",
           "properties": {
-            "file": {
-              "type": "string",
-              "format": "binary"
+            "file_name": {
+              "type": "string"
             },
+            "parts": {
+              "type": "number"
+            }
+          },
+          "required": [
+            "file_name",
+            "parts"
+          ]
+        },
+        "ICompleteUploadCSVFile": {
+          "type": "object",
+          "properties": {
+            "upload_id": {
+              "type": "string"
+            },
+            "file_name": {
+              "type": "string"
+            },
+            "parts": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "required": [
+            "upload_id",
+            "file_name",
+            "parts"
+          ]
+        },
+        "ICreatePipelineCSVFile": {
+          "type": "object",
+          "properties": {
             "name": {
               "type": "string"
             },
+            "file_name": {
+              "type": "string"
+            },
             "description": {
               "type": "string"
             },
@@ -5243,8 +5361,8 @@
             }
           },
           "required": [
-            "file",
             "name",
+            "file_name",
             "description",
             "encoding",
             "sep",

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.27.0",
+        "@dadosfera/protospack-v2": "3.28.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -31,7 +31,7 @@
         "dotenv": "^14.3.2",
         "elastic-apm-node": "^3.36.0",
         "helmet": "^5.1.0",
-        "jsonwebtoken": "^8.5.1",
+        "jsonwebtoken": "^9.0.0",
         "jwk-to-pem": "^2.0.5",
         "mixpanel": "^0.17.0",
         "passport": "^0.6.0",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.27.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
-      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
+      "version": "3.28.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz",
+      "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -7786,30 +7786,18 @@
       }
     },
     "node_modules/jsonwebtoken": {
-      "version": "8.5.1",
-      "license": "MIT",
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
+      "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
       "dependencies": {
         "jws": "^3.2.2",
-        "lodash.includes": "^4.3.0",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isinteger": "^4.0.4",
-        "lodash.isnumber": "^3.0.3",
-        "lodash.isplainobject": "^4.0.6",
-        "lodash.isstring": "^4.0.1",
-        "lodash.once": "^4.0.0",
+        "lodash": "^4.17.21",
         "ms": "^2.1.1",
-        "semver": "^5.6.0"
+        "semver": "^7.3.8"
       },
       "engines": {
-        "node": ">=4",
-        "npm": ">=1.4.28"
-      }
-    },
-    "node_modules/jsonwebtoken/node_modules/semver": {
-      "version": "5.7.1",
-      "license": "ISC",
-      "bin": {
-        "semver": "bin/semver"
+        "node": ">=12",
+        "npm": ">=6"
       }
     },
     "node_modules/jwa": {
@@ -7908,30 +7896,6 @@
       "version": "4.3.0",
       "license": "MIT"
     },
-    "node_modules/lodash.includes": {
-      "version": "4.3.0",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isboolean": {
-      "version": "3.0.3",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isinteger": {
-      "version": "4.0.4",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isnumber": {
-      "version": "3.0.3",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isplainobject": {
-      "version": "4.0.6",
-      "license": "MIT"
-    },
-    "node_modules/lodash.isstring": {
-      "version": "4.0.1",
-      "license": "MIT"
-    },
     "node_modules/lodash.memoize": {
       "version": "4.1.2",
       "dev": true,
@@ -7942,10 +7906,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "node_modules/lodash.once": {
-      "version": "4.1.1",
-      "license": "MIT"
-    },
     "node_modules/lodash.sortby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
@@ -9484,9 +9444,9 @@
       "license": "MIT"
     },
     "node_modules/semver": {
-      "version": "7.3.7",
-      "dev": true,
-      "license": "ISC",
+      "version": "7.3.8",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
+      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
       "dependencies": {
         "lru-cache": "^6.0.0"
       },
@@ -12324,9 +12284,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.27.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.27.0.tgz",
-      "integrity": "sha512-yI9KiO6KwhFTyOLJv/Gqf1HqaSQ0FokPhoQxP+RvJCKi62aW1mSkA4PyYVeQNnj9lzyn+mKjSvWWe9puFficNw==",
+      "version": "3.28.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.28.0.tgz",
+      "integrity": "sha512-JZYYhoaXFUpb5W/fBVt4XYe1Hlef2x9aLlM2Yv0erYn9QAu+/Pb99YnXbkgawQVlmJojSRrAvSwq1o9yOgKCrg==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -16455,23 +16415,14 @@
       }
     },
     "jsonwebtoken": {
-      "version": "8.5.1",
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.0.tgz",
+      "integrity": "sha512-tuGfYXxkQGDPnLJ7SibiQgVgeDgfbPq2k2ICcbgqW8WxWLBAxKQM/ZCu/IT8SOSwmaYl4dpTFCW5xZv7YbbWUw==",
       "requires": {
         "jws": "^3.2.2",
-        "lodash.includes": "^4.3.0",
-        "lodash.isboolean": "^3.0.3",
-        "lodash.isinteger": "^4.0.4",
-        "lodash.isnumber": "^3.0.3",
-        "lodash.isplainobject": "^4.0.6",
-        "lodash.isstring": "^4.0.1",
-        "lodash.once": "^4.0.0",
+        "lodash": "^4.17.21",
         "ms": "^2.1.1",
-        "semver": "^5.6.0"
-      },
-      "dependencies": {
-        "semver": {
-          "version": "5.7.1"
-        }
+        "semver": "^7.3.8"
       }
     },
     "jwa": {
@@ -16544,24 +16495,6 @@
     "lodash.camelcase": {
       "version": "4.3.0"
     },
-    "lodash.includes": {
-      "version": "4.3.0"
-    },
-    "lodash.isboolean": {
-      "version": "3.0.3"
-    },
-    "lodash.isinteger": {
-      "version": "4.0.4"
-    },
-    "lodash.isnumber": {
-      "version": "3.0.3"
-    },
-    "lodash.isplainobject": {
-      "version": "4.0.6"
-    },
-    "lodash.isstring": {
-      "version": "4.0.1"
-    },
     "lodash.memoize": {
       "version": "4.1.2",
       "dev": true
@@ -16570,9 +16503,6 @@
       "version": "4.6.2",
       "dev": true
     },
-    "lodash.once": {
-      "version": "4.1.1"
-    },
     "lodash.sortby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
@@ -17584,8 +17514,9 @@
       }
     },
     "semver": {
-      "version": "7.3.7",
-      "dev": true,
+      "version": "7.3.8",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz",
+      "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==",
       "requires": {
         "lru-cache": "^6.0.0"
       }

package.json

@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.27.0",
+    "@dadosfera/protospack-v2": "3.28.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",
@@ -47,7 +47,7 @@
     "dotenv": "^14.3.2",
     "elastic-apm-node": "^3.36.0",
     "helmet": "^5.1.0",
-    "jsonwebtoken": "^8.5.1",
+    "jsonwebtoken": "^9.0.0",
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
     "passport": "^0.6.0",

src/modules/auth/auth.controller.ts

@@ -29,7 +29,6 @@ import { AuthClientService } from './auth.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
 import { RequestUser, User } from 'src/authentication/user.decorator';
-import { Metadata } from '@grpc/grpc-js';
 import {
   AuthRefreshAccessTokenReq,
   AuthSignInReq,
@@ -58,8 +57,7 @@ export class AuthController {
     @Headers('Dadosfera-Lang') language: string,
   ): Promise<AuthSignInRes> {
     this.logger.info('/auth - SignIn');
-    const metadata = new Metadata();
-    metadata.add('language', language || 'pt-br');
+    const metadata = PackTheMetadata({ language: language || 'pt-br' });
     return this.authClient.signIn({ username, password, totp }, metadata);
   }
 
@@ -115,12 +113,18 @@ export class AuthController {
 
   @Post('reset-password')
   @HttpCode(HttpStatus.OK)
-  async resetPassword(@Body() body: AuthResetPasswordRequest) {
+  async resetPassword(
+    @Body() body: AuthResetPasswordRequest,
+    @Headers('dadosfera-lang') language,
+  ) {
     this.logger.info('/auth - reset-password');
+    const metadata = PackTheMetadata({
+      language: language,
+    });
 
     const { username } = body;
 
-    return this.authClient.resetPassword({ username });
+    return this.authClient.resetPassword({ username }, metadata);
   }
 
   @Post('verify-reset-password-code')

src/modules/auth/auth.service.ts

@@ -90,10 +90,15 @@ export class AuthClientService implements OnModuleInit {
     );
   }
 
-  async resetPassword({ username }: AuthResetPasswordRequest) {
+  async resetPassword(
+    { username }: AuthResetPasswordRequest,
+    metadata: Metadata,
+  ) {
     this.logger.info('resetPassword');
 
-    return lastValueFrom(this.authService.AuthResetPassword({ username }));
+    return lastValueFrom(
+      this.authService.AuthResetPassword({ username }, metadata),
+    );
   }
 
   async verifyResetPasswordCode({

src/modules/auth/dtos/login.ts

@@ -72,6 +72,8 @@ export class AuthCustomer {
   name: string;
   @ApiProperty()
   tier: string;
+  @ApiProperty()
+  scheduleLimit: string;
 }
 
 export class AuthSignInReq implements AuthSignInRequest {

src/modules/mixpanel/mixpanel.controller.ts

@@ -15,9 +15,55 @@ export class MixpanelController {
     delete body.info;
     const mixpanel = init(this.mixpanelToken);
 
+    const separator = user.username.includes('-') ? '-' : '.';
+    const removeValues = [
+      '.dadosferatech.dadosfera',
+      '.demo.dadosfera',
+      '.dadosferademo',
+      '.dadosferarh.dadosfera',
+      '.dadosferatech.dadosfera2',
+      '.dadosferatech.dadosfera',
+      '.dadosfera.fin',
+      '.dadosferafin.dadosfera',
+      '.praxio.dadosfera',
+      '.dadosfera.tech',
+      '.treinamentos@dadosfera.ai',
+      '.dadosfera2',
+      '.treinamentosfera',
+      '.dadosfera',
+    ];
+
+    let username = user.username;
+
+    removeValues.forEach((value) => {
+      username = username.replace(value, '');
+    });
+
+    username = username.split('@')?.[0];
+    username = username.split('+')?.[0];
+
+    let firstName = username
+      .substring(0, username.indexOf(separator))
+      .replace('dadosfera', '');
+    let lastName = username
+      .substring(username.lastIndexOf(separator) + 1)
+      .replace('dadosfera', '');
+
+    if (!firstName) {
+      firstName = lastName;
+      lastName = '';
+    }
+
+    firstName = this.capitalize(firstName);
+    lastName = this.capitalize(lastName);
+
     await mixpanel.people.set(user.username, {
-      $name: user.username,
-      $email: user.username,
+      $first_name: firstName,
+      $last_name: lastName,
+      $name: this.getFullName(firstName, lastName),
+      $email: user.username.includes('@')
+        ? user.username
+        : user.username + '@dadosfera.ai',
       customer_name: user.customer_name,
     });
 
@@ -30,4 +76,16 @@ export class MixpanelController {
 
     return { id, body, user: user.username };
   }
+
+  capitalize(sentence: string): string {
+    if (!sentence) {
+      return '';
+    }
+
+    return sentence[0].toUpperCase() + sentence.substring(1);
+  }
+
+  getFullName(firstName: string, lastName: string) {
+    return `${firstName}${lastName ? ' ' + lastName : ''}`;
+  }
 }

src/modules/oauth/oauth.controller.ts

@@ -148,10 +148,16 @@ export class OauthController {
     const { customer_id, customer_name, user_id } = connectionInfo;
 
     try {
-      const response = await this.connectionService.createConnection(
-        connectionInfo,
-        PackTheMetadata({ customer_id, customer_name, user_id }),
-      );
+      const response = connectionInfo.id
+        ? await this.connectionService.updateConnection(
+            connectionInfo.id,
+            connectionInfo,
+            PackTheMetadata({ customer_id, customer_name, user_id }),
+          )
+        : await this.connectionService.createConnection(
+            connectionInfo,
+            PackTheMetadata({ customer_id, customer_name, user_id }),
+          );
       url.searchParams.set('connection_id', response.connection.id);
       return { url: url.href };
     } catch (error) {

src/modules/oauth/oauth.service.ts

@@ -23,6 +23,7 @@ export class OauthService {
       user_id,
       customer_id,
       customer_name,
+      id,
     } = query;
 
     const properties = { plugin, credentials_type: 'oauth' };
@@ -45,6 +46,7 @@ export class OauthService {
       type,
       properties,
       plugin,
+      id,
     };
     return jwt.sign({ ...newConnection }, process.env.JWT_PRIVATE_KEY);
   }

src/modules/pipelinesV2/interfaces.ts

@@ -67,13 +67,32 @@ export interface IGetPipelineLogsRequest {
   details: string;
 }
 
-export class IUploadCSVFile {
-  @ApiProperty({ format: 'binary' })
-  file: string;
+export class IInitUploadCSVFile {
+  @ApiProperty()
+  file_name: string;
 
+  @ApiProperty()
+  parts: number;
+}
+
+export class ICompleteUploadCSVFile {
+  @ApiProperty()
+  upload_id: string;
+
+  @ApiProperty()
+  file_name: string;
+
+  @ApiProperty()
+  parts: { ETag: string; PartNumber: number }[];
+}
+
+export class ICreatePipelineCSVFile {
   @ApiProperty()
   name: string;
 
+  @ApiProperty()
+  file_name: string;
+
   @ApiProperty()
   description: string;
 

src/modules/pipelinesV2/pipelines.controller.ts

@@ -13,12 +13,9 @@ import {
   HttpCode,
   HttpStatus,
   Patch,
-  UseInterceptors,
-  UploadedFile,
   HttpException,
 } from '@nestjs/common';
 import {
-  ApiConsumes,
   ApiCreatedResponse,
   ApiNoContentResponse,
   ApiTags,
@@ -36,12 +33,13 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
 import {
+  ICompleteUploadCSVFile,
+  ICreatePipelineCSVFile,
   ICreatePipelineV2Req,
   IPipelineV2,
-  IUploadCSVFile,
+  IInitUploadCSVFile,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
-import { FileInterceptor } from '@nestjs/platform-express';
 
 @UseFilters(new GrpcToHttpExceptionFilter())
 @ApiTags('PipelinesV2')
@@ -300,37 +298,27 @@ export class PipelinesController {
     this.logger.info('PipelinesController - delete: OK');
   }
 
-  @Post('/upload')
+  @Post('/init-upload')
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
-  @ApiConsumes('multipart/form-data')
-  @UseInterceptors(
-    FileInterceptor('file', { limits: { fileSize: 50 * 1000 * 1000 + 1 } }),
-  )
-  async uploadFile(
+  async initUploadFile(
     @User() user: RequestUser,
-    @UploadedFile() file,
-    @Body() body: IUploadCSVFile,
+    @Body() body: IInitUploadCSVFile,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('/upload - Init Upload File Route');
     const metadata = PackTheMetadata({ ...user });
 
-    const parts_of_file_name = file.originalname.split('.');
-    const file_format = parts_of_file_name.pop();
-    const file_name = parts_of_file_name.join('.');
-    let name = `${new Date().getTime()}_${file_name}`;
-    body.name ? (name = `${new Date().getTime()}_${body.name}`) : name;
+    const { file_name, parts } = body;
 
-    const source_prefix = `${user.customer_name}/${name}.${file_format}`;
+    const { urls, upload_id } =
+      await this.pipelinesClientService.initUploadFile(
+        {
+          name: file_name,
+          parts: String(parts),
+        },
+        metadata,
+      );
 
-    const response = await this.pipelinesClientService.uploadFile(
-      {
-        file,
-        name,
-      },
-      metadata,
-    );
-
-    if (!response.url) {
+    if (!urls) {
       this.logger.info('pipeline/upload - Failed File pipeline');
       throw new HttpException(
         'Upload failed, try again in a few minutes, if the problem persists, contact support.',
@@ -338,7 +326,37 @@ export class PipelinesController {
       );
     }
 
-    const { sep, header, encoding, description } = body;
+    return { urls: JSON.parse(urls), upload_id };
+  }
+
+  @Post('/complete-upload')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  async completeUploadFile(
+    @User() user: RequestUser,
+    @Body() body: ICompleteUploadCSVFile,
+  ) {
+    this.logger.info('/upload - Complete Upload File Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const { upload_id, parts, file_name } = body;
+
+    return await this.pipelinesClientService.completeUploadFile(
+      { upload_id, parts, file_name },
+      metadata,
+    );
+  }
+
+  @Post('/file')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  async uploadedFile(
+    @User() user: RequestUser,
+    @Body() body: ICreatePipelineCSVFile,
+  ) {
+    this.logger.info('/upload - Upload Connector Route');
+    const metadata = PackTheMetadata({ ...user });
+
+    const { name, sep, header, encoding, description, file_name } = body;
+    const source_prefix = `${user.customer_name}/${file_name}`;
 
     const upload_pipeline = {
       connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
@@ -346,7 +364,7 @@ export class PipelinesController {
       connector_plugin: 'aws_s3',
       connector_version: '1.0.0',
       image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
-      name: body.name || file_name,
+      name,
       description,
       transformations_ids: [],
       tags: [],

src/modules/pipelinesV2/pipelines.service.ts

@@ -261,13 +261,10 @@ export class PipelinesService implements OnModuleInit {
     return { objects: JSON.parse(objects) };
   }
 
-  async uploadFile(uploadFile, metadata) {
+  async initUploadFile(uploadFile, metadata) {
     const body: Messages.PipelineV2UploadFileRequest = {
-      file: {
-        buffer: uploadFile.file.buffer,
-        mimetypes: uploadFile.file.mimetype,
-      },
       name: uploadFile.name,
+      parts: uploadFile.parts,
     };
 
     return lastValueFrom(
@@ -293,4 +290,21 @@ export class PipelinesService implements OnModuleInit {
 
     return url;
   }
+
+  async completeUploadFile(uploadFile, metadata) {
+    const body: Messages.PipelineV2CompleteUploadFileRequest = {
+      upload_id: uploadFile.upload_id,
+      parts: JSON.stringify(uploadFile.parts),
+      name: uploadFile.file_name,
+    };
+
+    return lastValueFrom(
+      this.pipelineWriteService.PipelineV2CompleteUploadFile(body, metadata),
+    ).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+  }
 }

src/modules/roles/dto/entities.ts

@@ -25,6 +25,8 @@ export class RoleDto {
   customer?: Customer;
   @ApiProperty()
   usage: string;
+  @ApiPropertyOptional({ description: 'Whether this role can be untoggled' })
+  canUntoggle?: boolean;
 }
 
 export class CustomerRole extends OmitType(RoleDto, ['customer']) {}
@@ -34,11 +36,17 @@ export enum TrueOrFalseEnum {
   true = 'true',
   false = 'false',
 }
+
+export enum CanUntoggleLogicEnum {
+  data_assets_sharing = 'data_assets_sharing',
+}
 export class GetRolesByCustomerReq {
   @ApiPropertyOptional()
   permissionId?: string[];
   @ApiPropertyOptional({ enum: TrueOrFalseEnum })
   getUsers?: TrueOrFalseEnum;
+  @ApiPropertyOptional({ enum: CanUntoggleLogicEnum })
+  canUntoggleLogic?: CanUntoggleLogicEnum;
 }
 
 export class GetRolesByCustomerRes {

src/modules/roles/roles.controller.ts

@@ -71,12 +71,12 @@ export class RolesController {
   @ApiOkResponse({ type: GetRolesByCustomerRes })
   async searchRoles(
     @User() user: RequestUser,
-    @Query() filters: GetRolesByCustomerReq,
+    @Query() params: GetRolesByCustomerReq,
     @Headers('dadosfera-lang') language,
   ) {
     this.logger.info('searchRoles', { user });
     this.rolesService.setLanguage(language);
-    return await this.rolesService.roleSearch(filters, user);
+    return await this.rolesService.roleSearch(params, user);
   }
 
   @Post()

src/modules/roles/roles.service.ts

@@ -70,12 +70,7 @@ export class RolesService {
     return { role: roleTreated };
   }
 
-  async roleSearch(
-    filters: GetRolesByCustomerReq,
-    { customer_id, access_token }: any,
-  ) {
-    const meta = new Metadata();
-    meta.add('access_token', access_token);
+  async roleSearch(filters: GetRolesByCustomerReq, { customer_id }) {
     let { permissionId: permissionIds } = filters;
     if (permissionIds && !Array.isArray(permissionIds))
       permissionIds = [permissionIds];
@@ -89,15 +84,13 @@ export class RolesService {
         return seqId;
       }) || [];
     const roles = await lastValueFrom(
-      this.rolesClientService.RoleSearch(
-        {
-          customerId: customer_id,
-          getUsers: isGetUsers,
-          permissionIds: [],
-          permissionSeqIds,
-        },
-        meta,
-      ),
+      this.rolesClientService.RoleSearch({
+        customerId: customer_id,
+        getUsers: isGetUsers,
+        permissionIds: [],
+        permissionSeqIds,
+        canUntoggleLogic: filters.canUntoggleLogic,
+      }),
     );
     const rolesTreated = this.getRolesPermissionsName(roles.roles);
     return { roles: rolesTreated };

src/modules/users/users.controller.ts

@@ -48,9 +48,9 @@ import {
   IUserByCustomer,
 } from './dtos/entities';
 import { UsersService } from './users.service';
-import { Metadata } from '@grpc/grpc-js';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
 import ErrorCodes from 'src/utils/errorCodes';
+import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 @ApiTags('Users')
 @Controller('users')
@@ -149,12 +149,14 @@ export class UsersController {
     @Body() body: CreateUserReq,
     @Headers('dadosfera-lang') language,
   ) {
-    const meta = new Metadata();
-    meta.add('access_token', user.access_token);
+    const metadata = PackTheMetadata({
+      access_token: user.access_token,
+      language,
+    });
     this.logger.info('createUser', { user });
     this.userService.setLanguage(language);
 
-    return await this.userService.createUser(body, meta);
+    return await this.userService.createUser(body, metadata);
   }
 
   @Post('batch')
@@ -165,11 +167,15 @@ export class UsersController {
     @Body() body: BatchCreateUserReq,
     @Headers('dadosfera-lang') language,
   ) {
-    const meta = new Metadata();
-    meta.add('access_token', user.access_token);
     this.logger.info('batchCreateUser', { user });
     this.userService.setLanguage(language);
-    return await this.userService.batchCreateUser(body, meta);
+
+    const metadata = PackTheMetadata({
+      access_token: user.access_token,
+      language,
+    });
+
+    return await this.userService.batchCreateUser(body, metadata);
   }
 
   @Post(':id/resend-invite')
@@ -182,7 +188,11 @@ export class UsersController {
   ) {
     this.logger.info('resendInvite', { user });
     this.userService.setLanguage(language);
-    return this.userService.resendInvite({ id });
+
+    const metadata = PackTheMetadata({
+      language,
+    });
+    return this.userService.resendInvite({ id }, metadata);
   }
 
   @Delete('role')

src/modules/users/users.service.ts

@@ -185,8 +185,10 @@ export class UsersService implements OnModuleInit {
     };
   }
 
-  async resendInvite(body: IdRequest) {
-    return lastValueFrom(this.usersClientService.UserResendInvite(body));
+  async resendInvite(body: IdRequest, metadata: Metadata) {
+    return lastValueFrom(
+      this.usersClientService.UserResendInvite(body, metadata),
+    );
   }
 
   async assignRoleToUser(body: AssignRoleToUserRequest) {

src/utils/ PackTheMetadata.ts

@@ -10,6 +10,7 @@ interface IMetadata {
   sensitive?: string;
   roles?: string[];
   is_data_manager?: boolean;
+  access_token?: string;
 }
 
 export function PackTheMetadata(info: IMetadata): Metadata {