FEAT: connector manager

package-lock.json

@@ -45,6 +45,7 @@
         "@types/jest": "27.0.2",
         "@types/jsonwebtoken": "^8.5.8",
         "@types/jwk-to-pem": "^2.0.1",
+        "@types/multer": "^1.4.7",
         "@types/node": "^16.11.26",
         "@types/supertest": "^2.0.12",
         "@typescript-eslint/eslint-plugin": "^5.17.0",
@@ -3031,6 +3032,15 @@
       "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==",
       "dev": true
     },
+    "node_modules/@types/multer": {
+      "version": "1.4.7",
+      "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.7.tgz",
+      "integrity": "sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==",
+      "dev": true,
+      "dependencies": {
+        "@types/express": "*"
+      }
+    },
     "node_modules/@types/node": {
       "version": "16.11.41",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz",
@@ -12712,6 +12722,15 @@
       "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==",
       "dev": true
     },
+    "@types/multer": {
+      "version": "1.4.7",
+      "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.7.tgz",
+      "integrity": "sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==",
+      "dev": true,
+      "requires": {
+        "@types/express": "*"
+      }
+    },
     "@types/node": {
       "version": "16.11.41",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz",

package.json

@@ -63,6 +63,7 @@
     "@types/jest": "27.0.2",
     "@types/jsonwebtoken": "^8.5.8",
     "@types/jwk-to-pem": "^2.0.1",
+    "@types/multer": "^1.4.7",
     "@types/node": "^16.11.26",
     "@types/supertest": "^2.0.12",
     "@typescript-eslint/eslint-plugin": "^5.17.0",

src/app.module.ts

@@ -34,12 +34,16 @@ import { OauthController } from './modules/oauth/oauth.controller';
 import { getOauthSecrets } from './utils/OauthSecrets';
 import { AuthenticationGuard } from './authentication/authentication.guard';
 import { APP_GUARD } from '@nestjs/core';
+import { ConnectorClientConfiguration } from './clients/connector/client.config';
+import { ConnectorController } from './modules/connector/connector.controller';
+import { ConnectorClientService } from './clients/connector/client.service';
 
 const ducClient = new DucClient();
 const inputClient = new InputsClientConfiguration();
 const outputClient = new OutputsClientConfiguration();
 const pipelineClient = new PipelinesClientConfiguration();
 const transformationClient = new TransformationsClientConfiguration();
+const connectorClient = new ConnectorClientConfiguration();
 
 @Module({
   controllers: [
@@ -51,6 +55,7 @@ const transformationClient = new TransformationsClientConfiguration();
     HealthController,
     CatalogController,
     OauthController,
+    ConnectorController,
   ],
   providers: [
     { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
@@ -58,6 +63,7 @@ const transformationClient = new TransformationsClientConfiguration();
     TransformationsService,
     OutputsService,
     PipelinesService,
+    ConnectorClientService,
     HealthService,
     InputsClientService,
     TransformationsClientService,
@@ -98,6 +104,10 @@ const transformationClient = new TransformationsClientConfiguration();
         name: 'PIPELINES_PACKAGE',
         ...pipelineClient.config(),
       },
+      {
+        name: 'CONNECTOR_PACKAGE',
+        ...connectorClient.config(),
+      },
     ]),
   ],
 })

src/clients/connector/client.config.ts

@@ -0,0 +1,31 @@
+import { credentials } from '@grpc/grpc-js';
+import { ClientOptions, Transport } from '@nestjs/microservices';
+import { ConnectorManager } from '@victorradael/protospack-v2';
+
+export class ConnectorClientConfiguration {
+  config(): ClientOptions {
+    return {
+      transport: Transport.GRPC,
+      options: {
+        url: process.env.INFACTORY_URL,
+        package: [
+          ConnectorManager.ProtoPackages.WritePackage,
+          ConnectorManager.ProtoPackages.ReadPackage,
+        ],
+        credentials:
+          process.env.LOCAL_ENV === 'local'
+            ? undefined
+            : credentials.createSsl(),
+        protoPath: [
+          ConnectorManager.ProtoPaths.WriteFilePath,
+          ConnectorManager.ProtoPaths.ReadFilePath,
+        ],
+        loader: {
+          enums: String,
+          objects: true,
+          arrays: true,
+        },
+      },
+    };
+  }
+}

src/clients/connector/client.service.ts

@@ -0,0 +1,266 @@
+import {
+  OnModuleInit,
+  Inject,
+  HttpException,
+  HttpStatus,
+} from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import { ConnectorManager } from '@victorradael/protospack-v2';
+import { from } from 'rxjs';
+
+export class ConnectorClientService implements OnModuleInit {
+  private connectorServiceRead: ConnectorManager.ReadService.ConnectorManagerReadServices;
+  private connectorServiceWrite: ConnectorManager.WriteService.ConnectorManagerWriteServices;
+  constructor(
+    @Inject('CONNECTOR_PACKAGE') private readonly grpcClient: ClientGrpc,
+  ) {}
+
+  onModuleInit() {
+    this.connectorServiceWrite =
+      this.grpcClient.getService<ConnectorManager.WriteService.ConnectorManagerWriteServices>(
+        ConnectorManager.ProtoServices.ConnectorManagerWriteServices,
+      );
+
+    this.connectorServiceRead =
+      this.grpcClient.getService<ConnectorManager.ReadService.ConnectorManagerReadServices>(
+        ConnectorManager.ProtoServices.ConnectorManagerReadServices,
+      );
+  }
+
+  async uploadConnector(uploadConnector) {
+    const connector: ConnectorManager.Entities.ConnectorCreateRequest = {
+      file: {
+        buffer: uploadConnector.file.buffer,
+        mimetypes: uploadConnector.file.mimetype,
+      },
+      connector: JSON.stringify(uploadConnector.connector),
+    };
+    const serviceBody: ConnectorManager.Messages.RegisterConnectorRequest = {
+      connector,
+    };
+
+    const observable = from(
+      this.connectorServiceWrite.RegisterConnector(serviceBody),
+    );
+
+    const result = await new Promise((resolve, reject) => {
+      observable.subscribe({
+        next(x) {
+          resolve(x);
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    }).catch((err) => {
+      console.log(err.details);
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+
+    console.log('ConnectorClientService', 'Upload');
+
+    return result;
+  }
+
+  async uploadFile(uploadFile) {
+    const body: ConnectorManager.Messages.UploadFileRequest = {
+      file: {
+        buffer: uploadFile.file.buffer,
+        mimetypes: uploadFile.file.mimetype,
+      },
+      name: uploadFile.name,
+    };
+
+    const observable = from(this.connectorServiceWrite.UploadFile(body));
+
+    const result = await new Promise((resolve, reject) => {
+      observable.subscribe({
+        next(x) {
+          resolve(x);
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    }).catch((err) => {
+      console.log(err.details);
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+
+    console.log('ConnectorClientService', 'Upload');
+
+    return result;
+  }
+
+  async getAllConnectors(body) {
+    const observable = from(
+      this.connectorServiceRead.GetAllConnectors({
+        search: body.search,
+        filters: JSON.stringify(body.filters),
+      }),
+    );
+    const result = await new Promise((resolve, reject) => {
+      observable.subscribe({
+        next(x) {
+          resolve(x);
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    }).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+
+    console.log('ConnectorClientService', 'getAllConnectors');
+
+    return result;
+  }
+
+  async getConnectorDetails(plugin: string) {
+    const observable = from(
+      this.connectorServiceRead.GetConnectorDetails({
+        plugin,
+      }),
+    );
+    const result = await new Promise((resolve, reject) => {
+      observable.subscribe({
+        next(x) {
+          resolve(x);
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    }).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+
+    console.log('ConnectorClientService', 'getAllConnectors');
+
+    return result;
+  }
+
+  async getConnector(plugin: string) {
+    const observable = from(
+      this.connectorServiceRead.GetConnector({
+        plugin,
+      }),
+    );
+    const result = await new Promise((resolve, reject) => {
+      observable.subscribe({
+        next(x) {
+          resolve(x);
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    }).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+
+    console.log('ConnectorClientService', 'getAllConnectors');
+
+    return result;
+  }
+
+  async deleteConnector(plugin: string, version: string) {
+    const observable = from(
+      this.connectorServiceWrite.DeleteConnector({
+        version,
+        plugin,
+      }),
+    );
+    const result = await new Promise((resolve, reject) => {
+      observable.subscribe({
+        next(x) {
+          resolve(x);
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    }).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+
+    console.log('ConnectorClientService', 'getAllConnectors');
+
+    return result;
+  }
+
+  async updateConnector({ plugin, changes }) {
+    const observable = from(
+      this.connectorServiceWrite.UpdateConnector({
+        plugin,
+        changes,
+      }),
+    );
+    const result = await new Promise((resolve, reject) => {
+      observable.subscribe({
+        next(x) {
+          resolve(x);
+        },
+        error(err) {
+          console.log('Observable Error');
+          reject(err);
+        },
+        complete() {
+          console.log('done');
+        },
+      });
+    }).catch((err) => {
+      throw new HttpException(
+        err.details,
+        err.code === 6 ? HttpStatus.CONFLICT : 400,
+      );
+    });
+
+    console.log('ConnectorClientService', 'getAllConnectors');
+
+    return result;
+  }
+}

src/main.ts

@@ -3,6 +3,8 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
 import { writeFileSync } from 'fs';
 import helmet from 'helmet';
 
+import documentEmpty from '../swagger_empty.json';
+
 import { AppModule } from './app.module';
 
 async function bootstrap() {
@@ -18,7 +20,7 @@ async function bootstrap() {
   app.use(helmet());
 
   const config = new DocumentBuilder()
-    .setTitle('Maestro Grpc Documentation')
+    .setTitle('Maestro')
     .setDescription('Documentation for Maestro gateway')
     .setVersion('1.0')
     .addBearerAuth()
@@ -29,6 +31,9 @@ async function bootstrap() {
   if (process.env.ENV != 'stg' && process.env.ENV != 'prd')
     SwaggerModule.setup('api', app, document);
 
+  const swaggerDoc = process.env.ENV === 'dev' ? document : documentEmpty;
+  SwaggerModule.setup('api', app, swaggerDoc);
+
   await app.listen(3333);
 }
 bootstrap();

src/modules/auth/auth.controller.ts

@@ -26,6 +26,7 @@ import { PermissionsClientService } from '../../clients/permissions/client.servi
 import { Permissions } from '../../authentication/permissions.enum';
 
 import ErrorBuilder from '../../utils/ErrorBuilder';
+import { LoginDto } from './dtos/login';
 
 @ApiTags('Auth')
 @Controller('auth')

src/modules/auth/dtos/login.ts

@@ -0,0 +1,12 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class LoginDto {
+  @ApiProperty()
+  username: string;
+
+  @ApiProperty()
+  password: string;
+
+  @ApiProperty({ required: false })
+  totp: string;
+}

src/modules/connector/connector.controller.ts

@@ -0,0 +1,210 @@
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  HttpException,
+  Param,
+  Put,
+  Post,
+  Query,
+  UploadedFile,
+  UseInterceptors,
+} from '@nestjs/common';
+import { FileInterceptor } from '@nestjs/platform-express';
+import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
+import { ConnectorClientService } from 'src/clients/connector/client.service';
+import { AddTagDto } from './dtos/add-tag';
+import { CreateConnectorDto } from './dtos/create-connector';
+import { DeleteConnectorDto } from './dtos/delete-connector';
+import { GetAllDto } from './dtos/get-all';
+import { RemoveTagDto } from './dtos/remove-tag';
+import { UpdateDto } from './dtos/update';
+import { UploadFileDto } from './dtos/upload-file';
+
+@ApiTags('connectors')
+@ApiBearerAuth()
+@Controller('connectors')
+export class ConnectorController {
+  constructor(private connectorClientService: ConnectorClientService) {}
+
+  @Post()
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FileInterceptor('file'))
+  async uploadConnector(
+    @UploadedFile() file,
+    @Body() body: CreateConnectorDto,
+  ) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const response = await this.connectorClientService.uploadConnector({
+      file,
+      connector: JSON.parse(body.connector),
+    });
+
+    return response;
+  }
+
+  @Post('/upload')
+  @ApiConsumes('multipart/form-data')
+  @UseInterceptors(FileInterceptor('file'))
+  async uploadFile(@UploadedFile() file, @Body() { name }: UploadFileDto) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const response = await this.connectorClientService.uploadFile({
+      file,
+      name,
+    });
+
+    return response;
+  }
+
+  @Get()
+  async getAllConnectors(@Query() queries: GetAllDto) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const { filters, search } = queries;
+
+    const response: any = await this.connectorClientService.getAllConnectors({
+      filters: filters || {},
+      search: search || '',
+    });
+
+    const connectors = JSON.parse(response.connectors).connectors;
+
+    return {
+      message: response.message,
+      connectors,
+    };
+  }
+
+  @Get('/:plugin')
+  async getConnector(
+    @Param('plugin') plugin: string,
+    @Query('version') version: string,
+  ) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const pluginId = `${plugin}-${version}`;
+
+    const response: any = await this.connectorClientService.getConnector(
+      pluginId,
+    );
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Get('/:plugin/details')
+  async getConnectorDetails(
+    @Param('plugin') plugin: string,
+    @Query('version') version: string,
+  ) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const pluginId = `${plugin}-${version}`;
+
+    const response: any = await this.connectorClientService.getConnectorDetails(
+      pluginId,
+    );
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Put('/:plugin')
+  @ApiConsumes('multipart/form-data')
+  async updateConnector(
+    @Param('plugin') plugin: string,
+    @Body() body: UpdateDto,
+  ) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const changes = body;
+
+    const response: any = await this.connectorClientService.updateConnector({
+      plugin,
+      changes: JSON.stringify(changes),
+    });
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Put('/:plugin/add-tag')
+  async addTagOnConnector(
+    @Param('plugin') plugin: string,
+    @Body() body: AddTagDto,
+  ) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const { tags } = body;
+
+    if (!tags || !(typeof tags === 'object') || !tags.length) {
+      throw new HttpException('Not found tags attributes', 400);
+    }
+
+    const changes = { plugin, tags };
+
+    const response: any = await this.connectorClientService.updateConnector({
+      plugin,
+      changes: JSON.stringify(changes),
+    });
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Put('/:plugin/remove-tag')
+  async removeTagOnConnector(
+    @Param('plugin') plugin: string,
+    @Body() body: RemoveTagDto,
+  ) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const { tags } = body;
+    const remove = tags;
+
+    if (!remove || !(typeof remove === 'object') || !remove.length) {
+      throw new HttpException('Not found remove attribute', 400);
+    }
+
+    const changes = { plugin, remove };
+
+    const response: any = await this.connectorClientService.updateConnector({
+      plugin,
+      changes: JSON.stringify(changes),
+    });
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+
+  @Delete('/:plugin')
+  async deleteConnector(
+    @Param('plugin') plugin: string,
+    @Body() { version }: DeleteConnectorDto,
+  ) {
+    console.log(`/upload`, 'Upload Connector Route');
+
+    const response: any = await this.connectorClientService.deleteConnector(
+      plugin,
+      version,
+    );
+
+    return {
+      message: response.message || 'ok',
+      connector: { ...JSON.parse(response.connector) },
+    };
+  }
+}

src/modules/connector/dtos/add-tag.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class AddTagDto {
+  @ApiProperty()
+  tags: string[];
+}

src/modules/connector/dtos/create-connector.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class CreateConnectorDto {
+  @ApiProperty({ format: 'binary' })
+  file: string;
+
+  @ApiProperty()
+  connector: string;
+}

src/modules/connector/dtos/delete-connector.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class DeleteConnectorDto {
+  @ApiProperty()
+  version: string;
+}

src/modules/connector/dtos/get-all.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class GetAllDto {
+  @ApiProperty()
+  search: string;
+
+  @ApiProperty()
+  filters: object;
+}

src/modules/connector/dtos/remove-tag.ts

@@ -0,0 +1,6 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class RemoveTagDto {
+  @ApiProperty()
+  tags: string[];
+}

src/modules/connector/dtos/update.ts

@@ -0,0 +1,42 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class UpdateDto {
+  @ApiProperty({ format: 'binary', required: false })
+  file?: string;
+
+  @ApiProperty()
+  version: string;
+
+  @ApiProperty({ required: false })
+  category?: string;
+
+  @ApiProperty({ required: false })
+  plugin?: string;
+
+  @ApiProperty({ required: false })
+  image?: string;
+
+  @ApiProperty({ required: false })
+  docs?: string;
+
+  @ApiProperty({ required: false })
+  name?: string;
+
+  @ApiProperty({
+    required: false,
+    description: 'skip_select_entities: boolean skip_select_columns: boolean;',
+  })
+  options?: {
+    skip_select_entities: boolean;
+    skip_select_columns: boolean;
+  };
+
+  @ApiProperty({
+    required: false,
+    type: 'array',
+    items: {
+      properties: {},
+    },
+  })
+  connection_controls?: object[];
+}

src/modules/connector/dtos/upload-file.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from '@nestjs/swagger';
+
+export class UploadFileDto {
+  @ApiProperty({ format: 'binary' })
+  file: string;
+
+  @ApiProperty()
+  name: string;
+}

swagger_empty.json

@@ -0,0 +1,17 @@
+{
+  "openapi": "3.0.0",
+  "paths": {
+  },
+  "info": {
+    "title": "404",
+    "description": "",
+    "version": "",
+    "contact": {}
+  },
+  "tags": [],
+  "servers": [],
+  "components": {
+    
+    "schemas": {}
+  }
+}

tsconfig.json

@@ -17,6 +17,7 @@
     "noImplicitAny": false,
     "strictBindCallApply": false,
     "forceConsistentCasingInFileNames": false,
-    "noFallthroughCasesInSwitch": false
+    "noFallthroughCasesInSwitch": false,
+    "resolveJsonModule": true
   }
 }