Merge pull request #150 from dadosfera/fix/bulk-connectors-upload

fix/bulk-connectors-upload

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.18.8",
+        "@dadosfera/protospack-v2": "3.18.10",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1726,9 +1726,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.18.8",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.8.tgz",
-      "integrity": "sha512-pzvuIzCrIf/ZG//lwMdCxtZ7w/ZhiLFBA3VWyN9uM7AjeqVl7LllHKV08Fw6YjNNGBmQF3DLJQTK8kMCQja4hA==",
+      "version": "3.18.10",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.10.tgz",
+      "integrity": "sha512-0bRUvo4p7cJJOL+B0hVTiSDaA2MGtirPkFVQW4lUfzsyY5vrsbAkMz6R3iPdPPwFSxGn5LwidHGkP3/l7hlEpA==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12306,9 +12306,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.18.8",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.8.tgz",
-      "integrity": "sha512-pzvuIzCrIf/ZG//lwMdCxtZ7w/ZhiLFBA3VWyN9uM7AjeqVl7LllHKV08Fw6YjNNGBmQF3DLJQTK8kMCQja4hA==",
+      "version": "3.18.10",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.18.10.tgz",
+      "integrity": "sha512-0bRUvo4p7cJJOL+B0hVTiSDaA2MGtirPkFVQW4lUfzsyY5vrsbAkMz6R3iPdPPwFSxGn5LwidHGkP3/l7hlEpA==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -27,7 +27,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.18.8",
+    "@dadosfera/protospack-v2": "3.18.10",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/app.module.ts

@@ -23,6 +23,7 @@ import { HealthModule } from './modules/health/health.module';
 import { CatalogModule } from './modules/catalog/catalog.module';
 import { ConfigModule } from '@nestjs/config';
 import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
+import { ProductboardModule } from './modules/productboard/productboard.module';
 
 @Module({
   controllers: [],
@@ -54,6 +55,7 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
     OauthModule,
     PipelinesV2Module,
     CatalogModule,
+    ProductboardModule,
   ],
 })
 export class AppModule {}

src/modules/catalog/catalog.controller.ts

@@ -12,7 +12,10 @@ import {
   Query,
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
-import { AuthenticateCondition } from '../../authentication/authentication.decorator';
+import {
+  AuthenticateCondition,
+  Authenticated,
+} from '../../authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -21,6 +24,7 @@ import { RequestUser, User } from 'src/authentication/user.decorator';
 
 @ApiTags('Catalog')
 @Controller('catalog')
+@Authenticated()
 @AuthenticateCondition((req, user) => {
   if (req.path.includes('/manage_permissions/')) return true;
   let action;

src/modules/connector/client.service.ts

@@ -71,27 +71,174 @@ export class ConnectorClientService implements OnModuleInit {
     );
   }
 
-  async uploadConnectorsFile(uploadFile) {
-    const connectors = JSON.parse(uploadFile.file.buffer.toString('utf8'));
-
-    const body = connectors.map((connector) => {
-      return JSON.stringify(connector);
-    });
-
-    const request = {
-      connectors: body,
-    };
-
-    return lastValueFrom(
-      this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage(
-        request,
-      ),
+  async uploadConnectorsFile(uploadFile: Express.Multer.File) {
+    const res = await lastValueFrom(
+      this.connectorServiceWrite.RegisterMultipleConnectorsWithoutImage({
+        connectors: uploadFile.buffer,
+      }),
     ).catch((err) => {
       throw new HttpException(
         err.details,
         err.code === 6 ? HttpStatus.CONFLICT : 400,
       );
     });
+    const responseParsed = JSON.parse(res.message);
+    const connectorErrors = responseParsed.items
+      .filter((i) => i.update._index === 'connector' && i.update.error)
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const connectionControlsErrors = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connection_controls' && i.update.error,
+      )
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const connectionStepsErrors = responseParsed.items
+      .filter((i) => i.update._index === 'connection_steps' && i.update.error)
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const configControlsErrors = responseParsed.items
+      .filter((i) => i.update._index === 'config_controls' && i.update.error)
+      .map((i) => ({
+        id: i.update._id,
+        reason: i.update.error,
+        status: i.error.status,
+      }));
+    const connectorCreated = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connector' && i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionControlsCreated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_controls' &&
+          i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionStepsCreated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_steps' &&
+          i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const configControlsCreated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'config_controls' &&
+          i.update.result === 'created',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectorUpdated = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connector' && i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionControlsUpdated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_controls' &&
+          i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionStepsUpdated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_steps' &&
+          i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const configControlsUpdated = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'config_controls' &&
+          i.update.result === 'updated',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectorNoop = responseParsed.items
+      .filter(
+        (i) => i.update._index === 'connector' && i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionControlsNoop = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_controls' &&
+          i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const connectionStepsNoop = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'connection_steps' && i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    const configControlsNoop = responseParsed.items
+      .filter(
+        (i) =>
+          i.update._index === 'config_controls' && i.update.result === 'noop',
+      )
+      .map((i) => ({
+        id: i.update._id,
+      }));
+    return {
+      errors: {
+        connectors: connectorErrors,
+        connection_controls: connectionControlsErrors,
+        connection_steps: connectionStepsErrors,
+        config_controls: configControlsErrors,
+      },
+      created: {
+        connectors: connectorCreated,
+        connection_controls: connectionControlsCreated,
+        connection_steps: connectionStepsCreated,
+        config_controls: configControlsCreated,
+      },
+      updated: {
+        connectors: connectorUpdated,
+        connection_controls: connectionControlsUpdated,
+        connection_steps: connectionStepsUpdated,
+        config_controls: configControlsUpdated,
+      },
+      noop: {
+        connectors: connectorNoop,
+        connection_controls: connectionControlsNoop,
+        connection_steps: connectionStepsNoop,
+        config_controls: configControlsNoop,
+      },
+    };
   }
 
   async getAllConnectors(body) {

src/modules/connector/connector.controller.ts

@@ -19,7 +19,6 @@ import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
 import { ConnectorClientService } from './client.service';
 import { AddTagDto } from './dtos/add-tag';
 import { CreateConnectorDto } from './dtos/create-connector';
-import { UploadConnectorsFileDto } from './dtos/create-multiple-connector';
 import { DeleteConnectorDto } from './dtos/delete-connector';
 import { GetAllDto } from './dtos/get-all';
 import { RemoveTagDto } from './dtos/remove-tag';
@@ -85,14 +84,10 @@ export class ConnectorController {
   @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.CREATE)
   @ApiConsumes('multipart/form-data')
   @UseInterceptors(FileInterceptor('file'))
-  async uploadConnectorsFile(@UploadedFile() file: UploadConnectorsFileDto) {
+  async uploadConnectorsFile(@UploadedFile() file: Express.Multer.File) {
     this.logger.info(`/upload`, 'Upload Connector Route');
 
-    const response = await this.connectorClientService.uploadConnectorsFile({
-      file,
-    });
-
-    return response;
+    return await this.connectorClientService.uploadConnectorsFile(file);
   }
 
   @Get()

src/modules/network-config/network-config.controller.ts

@@ -19,17 +19,18 @@ import { RequestUser, User } from 'src/authentication/user.decorator';
 import {
   CreateNetworkConfigReq,
   CreateNetworkConfigRes,
-  DeleteNetworkConfigRes,
   FindAllNetworkConfigsRes,
-  UpdateNetworkConfigReq,
-  UpdateNetworkConfigRes,
 } from './dto/network-config';
 import { FileInterceptor } from '@nestjs/platform-express';
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/authentication/authentication.decorator';
 
 const networkConfigPermissions = PERMISSIONS_GROUPS.NETWORK_CONFIG.permissions;
 @ApiTags('Network Config')
+@Authenticated()
 @Controller('network-config')
 export class NetworkConfigController {
   logger: any;

src/modules/oauth/oauth.controller.ts

@@ -5,7 +5,9 @@ import { ConnectionClientService } from '../connection/client.service';
 import jwt from 'jsonwebtoken';
 import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
+import { Authenticated } from 'src/authentication/authentication.decorator';
 @ApiTags('oauth')
+@Authenticated()
 @Controller('oauth')
 export class OauthController {
   redirectUrl: string;

src/modules/permissions/permissions.controller.ts

@@ -15,6 +15,7 @@ import { GetPublicPermissionsRes } from './dto/entities';
 import { PermissionsService } from './permissions.service';
 
 @ApiTags('Permissions')
+@Authenticated()
 @Controller('permissions')
 @ApiHeader({
   name: 'dadosfera-lang',
@@ -22,7 +23,6 @@ import { PermissionsService } from './permissions.service';
   required: false,
   description: 'Defaults to pt-br',
 })
-@Authenticated()
 export class PermissionsController {
   logger: DadosferaLogger;
 

src/modules/pipelines/pipelines.controller.ts

@@ -9,7 +9,10 @@ import {
   Put,
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
-import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
+import {
+  AuthenticateCondition,
+  Authenticated,
+} from 'src/authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -17,6 +20,7 @@ import { RequestUser, User } from 'src/authentication/user.decorator';
 
 @ApiTags('Pipelines')
 @Controller('pipelines')
+@Authenticated()
 @AuthenticateCondition((req, user) => {
   let action;
 

src/modules/productboard/dtos/index.ts

@@ -0,0 +1,12 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class INote {
+  @ApiProperty()
+  title: string;
+
+  @ApiProperty()
+  content: string;
+
+  @ApiProperty()
+  tags: string[];
+}

src/modules/productboard/productboard.controller.spec.ts

@@ -0,0 +1,19 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { ProductboardController } from './productboard.controller';
+
+describe('ProductboardController', () => {
+  let controller: ProductboardController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [ProductboardController],
+      providers: [],
+    }).compile();
+
+    controller = module.get<ProductboardController>(ProductboardController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

src/modules/productboard/productboard.controller.ts

@@ -0,0 +1,51 @@
+import { Body, Controller, HttpException, Post } from '@nestjs/common';
+import { ApiTags } from '@nestjs/swagger';
+import axios from 'axios';
+import { User, RequestUser } from 'src/authentication/user.decorator';
+import { getSecreteFromSecreteManager } from 'src/utils/SecretManager';
+import { INote } from './dtos';
+
+@ApiTags('Productboard')
+@Controller('productboard')
+export class ProductboardController {
+  @Post('notes')
+  async sendNote(@Body() note: INote, @User() user: RequestUser) {
+    const { title, content, tags } = note;
+    const { username, customer_name } = user;
+
+    const email = username.includes('@')
+      ? username
+      : username + `@${customer_name}.default`;
+
+    const path = process.env.PB_TOKEN_PATH;
+    const token = await getSecreteFromSecreteManager(path);
+
+    const response = await axios
+      .post(
+        `https://api.productboard.com/notes`,
+        {
+          title,
+          content,
+          tags,
+          user: {
+            email,
+          },
+        },
+        {
+          headers: {
+            Authorization: 'Bearer ' + token,
+          },
+        },
+      )
+      .catch(() => null);
+
+    if (!response) {
+      throw new HttpException(
+        'Your request could not be submitted, please try again in a few moments or contact support.',
+        502,
+      );
+    }
+
+    return response.data;
+  }
+}

src/modules/productboard/productboard.module.ts

@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common';
+import { ProductboardController } from './productboard.controller';
+
+@Module({
+  controllers: [ProductboardController],
+  providers: [],
+})
+export class ProductboardModule {}

src/modules/roles/roles.controller.ts

@@ -53,8 +53,8 @@ import { LanguageEnum } from 'src/utils/languages.enum';
   required: false,
   description: 'Defaults to pt-br',
 })
-@UseFilters(new GrpcToHttpExceptionFilter())
 @Authenticated()
+@UseFilters(new GrpcToHttpExceptionFilter())
 @Controller('roles')
 export class RolesController {
   logger: DadosferaLogger;

src/modules/users/users.controller.ts

@@ -21,7 +21,10 @@ import {
   ApiTags,
 } from '@nestjs/swagger';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
+import {
+  Authenticated,
+  RequireAllPermissions,
+} from 'src/authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@@ -56,6 +59,7 @@ import { Metadata } from '@grpc/grpc-js';
   required: false,
   description: 'Defaults to pt-br',
 })
+@Authenticated()
 @UseFilters(GrpcToHttpExceptionFilter)
 export class UsersController {
   logger: DadosferaLogger;

src/utils/SecretManager.ts

@@ -0,0 +1,17 @@
+import {
+  SecretsManagerClient,
+  GetSecretValueCommand,
+} from '@aws-sdk/client-secrets-manager';
+
+export async function getSecreteFromSecreteManager(path: string) {
+  const secretsManagerClient = new SecretsManagerClient({});
+
+  const getSecretComand = new GetSecretValueCommand({
+    SecretId: `${path}`,
+  });
+  const res = await secretsManagerClient
+    .send(getSecretComand)
+    .catch(() => null);
+
+  return res?.SecretString;
+}