Merge pull request #150 from dadosfera/fix/bulk-connectors-upload

fix/bulk-connectors-upload

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.18.8",
+        "@dadosfera/protospack-v2": "3.18.10",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1726,9 +1726,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.18.8",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.8.tgz",
-      "integrity": "sha512-pzvuIzCrIf/ZG//lwMdCxtZ7w/ZhiLFBA3VWyN9uM7AjeqVl7LllHKV08Fw6YjNNGBmQF3DLJQTK8kMCQja4hA==",
+      "version": "3.18.10",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.10.tgz",
+      "integrity": "sha512-0bRUvo4p7cJJOL+B0hVTiSDaA2MGtirPkFVQW4lUfzsyY5vrsbAkMz6R3iPdPPwFSxGn5LwidHGkP3/l7hlEpA==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12306,9 +12306,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.18.8",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.8.tgz",
-      "integrity": "sha512-pzvuIzCrIf/ZG//lwMdCxtZ7w/ZhiLFBA3VWyN9uM7AjeqVl7LllHKV08Fw6YjNNGBmQF3DLJQTK8kMCQja4hA==",
+      "version": "3.18.10",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.10.tgz",
+      "integrity": "sha512-0bRUvo4p7cJJOL+B0hVTiSDaA2MGtirPkFVQW4lUfzsyY5vrsbAkMz6R3iPdPPwFSxGn5LwidHGkP3/l7hlEpA==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -27,7 +27,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.18.8",
+    "@dadosfera/protospack-v2": "3.18.10",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/modules/connector/client.service.ts

@@ -71,27 +71,174 @@ export class ConnectorClientService implements OnModuleInit {
     );
   }
 
-  async uploadConnectorsFile(uploadFile) {
-    const connectors = JSON.parse(uploadFile.file.buffer.toString('utf8'));
-
-    const body = connectors.map((connector) => {
-      return JSON.stringify(connector);
-    });
-
-    const request = {
-      connectors: body,
-    };
-
-    return lastValueFrom(
-      this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage(
-        request,
-      ),
+  async uploadConnectorsFile(uploadFile: Express.Multer.File) {
+    const res = await lastValueFrom(
+      this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage({
+        connectors: uploadFile.buffer,
+      }),
     ).catch((err) => {
       throw new HttpException(
         err.details,
         err.code === 6 ? HttpStatus.CONFLICT : 400,
       );
     });
+    const responseParsed = JSON.parse(res.message);
+    const connectorErrors = responseParsed.items
+      .filter((i) => i.update._index === 'connector' && i.update.error)
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const connectionControlsErrors = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connection_controls' && i.update.error,
+      )
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const connectionStepsErrors = responseParsed.items
+      .filter((i) => i.update._index === 'connection_steps' && i.update.error)
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const configControlsErrors = responseParsed.items
+      .filter((i) => i.update._index === 'config_controls' && i.update.error)
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const connectorCreated = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connector' && i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionControlsCreated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_controls' &&
+          i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionStepsCreated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_steps' &&
+          i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const configControlsCreated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'config_controls' &&
+          i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectorUpdated = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connector' && i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionControlsUpdated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_controls' &&
+          i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionStepsUpdated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_steps' &&
+          i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const configControlsUpdated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'config_controls' &&
+          i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectorNoop = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connector' && i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionControlsNoop = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_controls' &&
+          i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionStepsNoop = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_steps' && i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const configControlsNoop = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'config_controls' && i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    return {
+      errors: {
+        connectors: connectorErrors,
+        connection_controls: connectionControlsErrors,
+        connection_steps: connectionStepsErrors,
+        config_controls: configControlsErrors,
+      },
+      created: {
+        connectors: connectorCreated,
+        connection_controls: connectionControlsCreated,
+        connection_steps: connectionStepsCreated,
+        config_controls: configControlsCreated,
+      },
+      updated: {
+        connectors: connectorUpdated,
+        connection_controls: connectionControlsUpdated,
+        connection_steps: connectionStepsUpdated,
+        config_controls: configControlsUpdated,
+      },
+      noop: {
+        connectors: connectorNoop,
+        connection_controls: connectionControlsNoop,
+        connection_steps: connectionStepsNoop,
+        config_controls: configControlsNoop,
+      },
+    };
   }
 
   async getAllConnectors(body) {

src/modules/connector/connector.controller.ts

@@ -19,7 +19,6 @@ import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
 import { ConnectorClientService } from './client.service';
 import { AddTagDto } from './dtos/add-tag';
 import { CreateConnectorDto } from './dtos/create-connector';
-import { UploadConnectorsFileDto } from './dtos/create-multiple-connector';
 import { DeleteConnectorDto } from './dtos/delete-connector';
 import { GetAllDto } from './dtos/get-all';
 import { RemoveTagDto } from './dtos/remove-tag';
@@ -85,14 +84,10 @@ export class ConnectorController {
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
   @ApiConsumes('multipart/form-data')
   @UseInterceptors(FileInterceptor('file'))
-  async uploadConnectorsFile(@UploadedFile() file: UploadConnectorsFileDto) {
+  async uploadConnectorsFile(@UploadedFile() file: Express.Multer.File) {
     this.logger.info(`/upload`, 'Upload Connector Route');
 
-    const response = await this.connectorClientService.uploadConnectorsFile({
-      file,
-    });
-
-    return response;
+    return await this.connectorClientService.uploadConnectorsFile(file);
   }
 
   @Get()