diff --git a/Insomnia_2022-06-23.zip b/Insomnia_2022-06-23.zip new file mode 100644 index 0000000..7103d04 Binary files /dev/null and b/Insomnia_2022-06-23.zip differ diff --git a/package-lock.json b/package-lock.json index 7c211e3..2d59b43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,6 +45,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.8", "@types/jwk-to-pem": "^2.0.1", + "@types/multer": "^1.4.7", "@types/node": "^16.11.26", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "^5.17.0", @@ -3031,6 +3032,15 @@ "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==", "dev": true }, + "node_modules/@types/multer": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.7.tgz", + "integrity": "sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==", + "dev": true, + "dependencies": { + "@types/express": "*" + } + }, "node_modules/@types/node": { "version": "16.11.41", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz", @@ -12712,6 +12722,15 @@ "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==", "dev": true }, + "@types/multer": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.7.tgz", + "integrity": "sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==", + "dev": true, + "requires": { + "@types/express": "*" + } + }, "@types/node": { "version": "16.11.41", "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.41.tgz", diff --git a/package.json b/package.json index 03e0132..6c7ccc7 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.8", "@types/jwk-to-pem": "^2.0.1", + "@types/multer": "^1.4.7", "@types/node": "^16.11.26", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "^5.17.0", diff --git a/src/app.module.ts b/src/app.module.ts index f511377..9e5e353 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -34,12 +34,16 @@ import { OauthController } from './modules/oauth/oauth.controller'; import { getOauthSecrets } from './utils/OauthSecrets'; import { AuthenticationGuard } from './authentication/authentication.guard'; import { APP_GUARD } from '@nestjs/core'; +import { ConnectorClientConfiguration } from './clients/connector/client.config'; +import { ConnectorController } from './modules/connector/connector.controller'; +import { ConnectorClientService } from './clients/connector/client.service'; const ducClient = new DucClient(); const inputClient = new InputsClientConfiguration(); const outputClient = new OutputsClientConfiguration(); const pipelineClient = new PipelinesClientConfiguration(); const transformationClient = new TransformationsClientConfiguration(); +const connectorClient = new ConnectorClientConfiguration(); @Module({ controllers: [ @@ -51,6 +55,7 @@ const transformationClient = new TransformationsClientConfiguration(); HealthController, CatalogController, OauthController, + ConnectorController, ], providers: [ { provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() }, @@ -58,6 +63,7 @@ const transformationClient = new TransformationsClientConfiguration(); TransformationsService, OutputsService, PipelinesService, + ConnectorClientService, HealthService, InputsClientService, TransformationsClientService, @@ -98,6 +104,10 @@ const transformationClient = new TransformationsClientConfiguration(); name: 'PIPELINES_PACKAGE', ...pipelineClient.config(), }, + { + name: 'CONNECTOR_PACKAGE', + ...connectorClient.config(), + }, ]), ], }) diff --git a/src/clients/connector/client.config.ts b/src/clients/connector/client.config.ts new file mode 100644 index 0000000..61db9a1 --- /dev/null +++ b/src/clients/connector/client.config.ts @@ -0,0 +1,31 @@ +import { credentials } from '@grpc/grpc-js'; +import { ClientOptions, Transport } from '@nestjs/microservices'; +import { ConnectorManager } from '@victorradael/protospack-v2'; + +export class ConnectorClientConfiguration { + config(): ClientOptions { + return { + transport: Transport.GRPC, + options: { + url: process.env.INFACTORY_URL, + package: [ + ConnectorManager.ProtoPackages.WritePackage, + ConnectorManager.ProtoPackages.ReadPackage, + ], + credentials: + process.env.LOCAL_ENV === 'local' + ? undefined + : credentials.createSsl(), + protoPath: [ + ConnectorManager.ProtoPaths.WriteFilePath, + ConnectorManager.ProtoPaths.ReadFilePath, + ], + loader: { + enums: String, + objects: true, + arrays: true, + }, + }, + }; + } +} diff --git a/src/clients/connector/client.service.ts b/src/clients/connector/client.service.ts new file mode 100644 index 0000000..e9b4ed1 --- /dev/null +++ b/src/clients/connector/client.service.ts @@ -0,0 +1,266 @@ +import { + OnModuleInit, + Inject, + HttpException, + HttpStatus, +} from '@nestjs/common'; +import { ClientGrpc } from '@nestjs/microservices'; +import { ConnectorManager } from '@victorradael/protospack-v2'; +import { from } from 'rxjs'; + +export class ConnectorClientService implements OnModuleInit { + private connectorServiceRead: ConnectorManager.ReadService.ConnectorManagerReadServices; + private connectorServiceWrite: ConnectorManager.WriteService.ConnectorManagerWriteServices; + constructor( + @Inject('CONNECTOR_PACKAGE') private readonly grpcClient: ClientGrpc, + ) {} + + onModuleInit() { + this.connectorServiceWrite = + this.grpcClient.getService( + ConnectorManager.ProtoServices.ConnectorManagerWriteServices, + ); + + this.connectorServiceRead = + this.grpcClient.getService( + ConnectorManager.ProtoServices.ConnectorManagerReadServices, + ); + } + + async uploadConnector(uploadConnector) { + const connector: ConnectorManager.Entities.ConnectorCreateRequest = { + file: { + buffer: uploadConnector.file.buffer, + mimetypes: uploadConnector.file.mimetype, + }, + connector: JSON.stringify(uploadConnector.connector), + }; + const serviceBody: ConnectorManager.Messages.RegisterConnectorRequest = { + connector, + }; + + const observable = from( + this.connectorServiceWrite.RegisterConnector(serviceBody), + ); + + const result = await new Promise((resolve, reject) => { + observable.subscribe({ + next(x) { + resolve(x); + }, + error(err) { + console.log('Observable Error'); + reject(err); + }, + complete() { + console.log('done'); + }, + }); + }).catch((err) => { + console.log(err.details); + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + + console.log('ConnectorClientService', 'Upload'); + + return result; + } + + async uploadFile(uploadFile) { + const body: ConnectorManager.Messages.UploadFileRequest = { + file: { + buffer: uploadFile.file.buffer, + mimetypes: uploadFile.file.mimetype, + }, + name: uploadFile.name, + }; + + const observable = from(this.connectorServiceWrite.UploadFile(body)); + + const result = await new Promise((resolve, reject) => { + observable.subscribe({ + next(x) { + resolve(x); + }, + error(err) { + console.log('Observable Error'); + reject(err); + }, + complete() { + console.log('done'); + }, + }); + }).catch((err) => { + console.log(err.details); + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + + console.log('ConnectorClientService', 'Upload'); + + return result; + } + + async getAllConnectors(body) { + const observable = from( + this.connectorServiceRead.GetAllConnectors({ + search: body.search, + filters: JSON.stringify(body.filters), + }), + ); + const result = await new Promise((resolve, reject) => { + observable.subscribe({ + next(x) { + resolve(x); + }, + error(err) { + console.log('Observable Error'); + reject(err); + }, + complete() { + console.log('done'); + }, + }); + }).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + + console.log('ConnectorClientService', 'getAllConnectors'); + + return result; + } + + async getConnectorDetails(plugin: string) { + const observable = from( + this.connectorServiceRead.GetConnectorDetails({ + plugin, + }), + ); + const result = await new Promise((resolve, reject) => { + observable.subscribe({ + next(x) { + resolve(x); + }, + error(err) { + console.log('Observable Error'); + reject(err); + }, + complete() { + console.log('done'); + }, + }); + }).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + + console.log('ConnectorClientService', 'getAllConnectors'); + + return result; + } + + async getConnector(plugin: string) { + const observable = from( + this.connectorServiceRead.GetConnector({ + plugin, + }), + ); + const result = await new Promise((resolve, reject) => { + observable.subscribe({ + next(x) { + resolve(x); + }, + error(err) { + console.log('Observable Error'); + reject(err); + }, + complete() { + console.log('done'); + }, + }); + }).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + + console.log('ConnectorClientService', 'getAllConnectors'); + + return result; + } + + async deleteConnector(plugin: string, version: string) { + const observable = from( + this.connectorServiceWrite.DeleteConnector({ + version, + plugin, + }), + ); + const result = await new Promise((resolve, reject) => { + observable.subscribe({ + next(x) { + resolve(x); + }, + error(err) { + console.log('Observable Error'); + reject(err); + }, + complete() { + console.log('done'); + }, + }); + }).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + + console.log('ConnectorClientService', 'getAllConnectors'); + + return result; + } + + async updateConnector({ plugin, changes }) { + const observable = from( + this.connectorServiceWrite.UpdateConnector({ + plugin, + changes, + }), + ); + const result = await new Promise((resolve, reject) => { + observable.subscribe({ + next(x) { + resolve(x); + }, + error(err) { + console.log('Observable Error'); + reject(err); + }, + complete() { + console.log('done'); + }, + }); + }).catch((err) => { + throw new HttpException( + err.details, + err.code === 6 ? HttpStatus.CONFLICT : 400, + ); + }); + + console.log('ConnectorClientService', 'getAllConnectors'); + + return result; + } +} diff --git a/src/main.ts b/src/main.ts index fec1592..71690c3 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,6 +3,8 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; import { writeFileSync } from 'fs'; import helmet from 'helmet'; +import documentEmpty from '../swagger_empty.json'; + import { AppModule } from './app.module'; async function bootstrap() { @@ -18,7 +20,7 @@ async function bootstrap() { app.use(helmet()); const config = new DocumentBuilder() - .setTitle('Maestro Grpc Documentation') + .setTitle('Maestro') .setDescription('Documentation for Maestro gateway') .setVersion('1.0') .addBearerAuth() @@ -29,6 +31,9 @@ async function bootstrap() { if (process.env.ENV != 'stg' && process.env.ENV != 'prd') SwaggerModule.setup('api', app, document); + const swaggerDoc = process.env.ENV === 'dev' ? document : documentEmpty; + SwaggerModule.setup('api', app, swaggerDoc); + await app.listen(3333); } bootstrap(); diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 62709eb..97e1794 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -26,6 +26,7 @@ import { PermissionsClientService } from '../../clients/permissions/client.servi import { Permissions } from '../../authentication/permissions.enum'; import ErrorBuilder from '../../utils/ErrorBuilder'; +import { LoginDto } from './dtos/login'; @ApiTags('Auth') @Controller('auth') diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts new file mode 100644 index 0000000..cae27fd --- /dev/null +++ b/src/modules/auth/dtos/login.ts @@ -0,0 +1,12 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class LoginDto { + @ApiProperty() + username: string; + + @ApiProperty() + password: string; + + @ApiProperty({ required: false }) + totp: string; +} diff --git a/src/modules/connector/connector.controller.ts b/src/modules/connector/connector.controller.ts new file mode 100644 index 0000000..e5bf810 --- /dev/null +++ b/src/modules/connector/connector.controller.ts @@ -0,0 +1,210 @@ +import { + Body, + Controller, + Delete, + Get, + HttpException, + Param, + Put, + Post, + Query, + UploadedFile, + UseInterceptors, +} from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger'; +import { ConnectorClientService } from 'src/clients/connector/client.service'; +import { AddTagDto } from './dtos/add-tag'; +import { CreateConnectorDto } from './dtos/create-connector'; +import { DeleteConnectorDto } from './dtos/delete-connector'; +import { GetAllDto } from './dtos/get-all'; +import { RemoveTagDto } from './dtos/remove-tag'; +import { UpdateDto } from './dtos/update'; +import { UploadFileDto } from './dtos/upload-file'; + +@ApiTags('connectors') +@ApiBearerAuth() +@Controller('connectors') +export class ConnectorController { + constructor(private connectorClientService: ConnectorClientService) {} + + @Post() + @ApiConsumes('multipart/form-data') + @UseInterceptors(FileInterceptor('file')) + async uploadConnector( + @UploadedFile() file, + @Body() body: CreateConnectorDto, + ) { + console.log(`/upload`, 'Upload Connector Route'); + + const response = await this.connectorClientService.uploadConnector({ + file, + connector: JSON.parse(body.connector), + }); + + return response; + } + + @Post('/upload') + @ApiConsumes('multipart/form-data') + @UseInterceptors(FileInterceptor('file')) + async uploadFile(@UploadedFile() file, @Body() { name }: UploadFileDto) { + console.log(`/upload`, 'Upload Connector Route'); + + const response = await this.connectorClientService.uploadFile({ + file, + name, + }); + + return response; + } + + @Get() + async getAllConnectors(@Query() queries: GetAllDto) { + console.log(`/upload`, 'Upload Connector Route'); + + const { filters, search } = queries; + + const response: any = await this.connectorClientService.getAllConnectors({ + filters: filters || {}, + search: search || '', + }); + + const connectors = JSON.parse(response.connectors).connectors; + + return { + message: response.message, + connectors, + }; + } + + @Get('/:plugin') + async getConnector( + @Param('plugin') plugin: string, + @Query('version') version: string, + ) { + console.log(`/upload`, 'Upload Connector Route'); + + const pluginId = `${plugin}-${version}`; + + const response: any = await this.connectorClientService.getConnector( + pluginId, + ); + + return { + message: response.message || 'ok', + connector: { ...JSON.parse(response.connector) }, + }; + } + + @Get('/:plugin/details') + async getConnectorDetails( + @Param('plugin') plugin: string, + @Query('version') version: string, + ) { + console.log(`/upload`, 'Upload Connector Route'); + + const pluginId = `${plugin}-${version}`; + + const response: any = await this.connectorClientService.getConnectorDetails( + pluginId, + ); + + return { + message: response.message || 'ok', + connector: { ...JSON.parse(response.connector) }, + }; + } + + @Put('/:plugin') + @ApiConsumes('multipart/form-data') + async updateConnector( + @Param('plugin') plugin: string, + @Body() body: UpdateDto, + ) { + console.log(`/upload`, 'Upload Connector Route'); + + const changes = body; + + const response: any = await this.connectorClientService.updateConnector({ + plugin, + changes: JSON.stringify(changes), + }); + + return { + message: response.message || 'ok', + connector: { ...JSON.parse(response.connector) }, + }; + } + + @Put('/:plugin/add-tag') + async addTagOnConnector( + @Param('plugin') plugin: string, + @Body() body: AddTagDto, + ) { + console.log(`/upload`, 'Upload Connector Route'); + + const { tags } = body; + + if (!tags || !(typeof tags === 'object') || !tags.length) { + throw new HttpException('Not found tags attributes', 400); + } + + const changes = { plugin, tags }; + + const response: any = await this.connectorClientService.updateConnector({ + plugin, + changes: JSON.stringify(changes), + }); + + return { + message: response.message || 'ok', + connector: { ...JSON.parse(response.connector) }, + }; + } + + @Put('/:plugin/remove-tag') + async removeTagOnConnector( + @Param('plugin') plugin: string, + @Body() body: RemoveTagDto, + ) { + console.log(`/upload`, 'Upload Connector Route'); + + const { tags } = body; + const remove = tags; + + if (!remove || !(typeof remove === 'object') || !remove.length) { + throw new HttpException('Not found remove attribute', 400); + } + + const changes = { plugin, remove }; + + const response: any = await this.connectorClientService.updateConnector({ + plugin, + changes: JSON.stringify(changes), + }); + + return { + message: response.message || 'ok', + connector: { ...JSON.parse(response.connector) }, + }; + } + + @Delete('/:plugin') + async deleteConnector( + @Param('plugin') plugin: string, + @Body() { version }: DeleteConnectorDto, + ) { + console.log(`/upload`, 'Upload Connector Route'); + + const response: any = await this.connectorClientService.deleteConnector( + plugin, + version, + ); + + return { + message: response.message || 'ok', + connector: { ...JSON.parse(response.connector) }, + }; + } +} diff --git a/src/modules/connector/dtos/add-tag.ts b/src/modules/connector/dtos/add-tag.ts new file mode 100644 index 0000000..7c04a23 --- /dev/null +++ b/src/modules/connector/dtos/add-tag.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class AddTagDto { + @ApiProperty() + tags: string[]; +} diff --git a/src/modules/connector/dtos/create-connector.ts b/src/modules/connector/dtos/create-connector.ts new file mode 100644 index 0000000..2cdc4d6 --- /dev/null +++ b/src/modules/connector/dtos/create-connector.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class CreateConnectorDto { + @ApiProperty({ format: 'binary' }) + file: string; + + @ApiProperty() + connector: string; +} diff --git a/src/modules/connector/dtos/delete-connector.ts b/src/modules/connector/dtos/delete-connector.ts new file mode 100644 index 0000000..6db5547 --- /dev/null +++ b/src/modules/connector/dtos/delete-connector.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class DeleteConnectorDto { + @ApiProperty() + version: string; +} diff --git a/src/modules/connector/dtos/get-all.ts b/src/modules/connector/dtos/get-all.ts new file mode 100644 index 0000000..f5e3469 --- /dev/null +++ b/src/modules/connector/dtos/get-all.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class GetAllDto { + @ApiProperty() + search: string; + + @ApiProperty() + filters: object; +} diff --git a/src/modules/connector/dtos/remove-tag.ts b/src/modules/connector/dtos/remove-tag.ts new file mode 100644 index 0000000..9de619b --- /dev/null +++ b/src/modules/connector/dtos/remove-tag.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class RemoveTagDto { + @ApiProperty() + tags: string[]; +} diff --git a/src/modules/connector/dtos/update.ts b/src/modules/connector/dtos/update.ts new file mode 100644 index 0000000..5262fde --- /dev/null +++ b/src/modules/connector/dtos/update.ts @@ -0,0 +1,42 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class UpdateDto { + @ApiProperty({ format: 'binary', required: false }) + file?: string; + + @ApiProperty() + version: string; + + @ApiProperty({ required: false }) + category?: string; + + @ApiProperty({ required: false }) + plugin?: string; + + @ApiProperty({ required: false }) + image?: string; + + @ApiProperty({ required: false }) + docs?: string; + + @ApiProperty({ required: false }) + name?: string; + + @ApiProperty({ + required: false, + description: 'skip_select_entities: boolean skip_select_columns: boolean;', + }) + options?: { + skip_select_entities: boolean; + skip_select_columns: boolean; + }; + + @ApiProperty({ + required: false, + type: 'array', + items: { + properties: {}, + }, + }) + connection_controls?: object[]; +} diff --git a/src/modules/connector/dtos/upload-file.ts b/src/modules/connector/dtos/upload-file.ts new file mode 100644 index 0000000..d167f3f --- /dev/null +++ b/src/modules/connector/dtos/upload-file.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class UploadFileDto { + @ApiProperty({ format: 'binary' }) + file: string; + + @ApiProperty() + name: string; +} diff --git a/swagger_empty.json b/swagger_empty.json new file mode 100644 index 0000000..d0eb1b4 --- /dev/null +++ b/swagger_empty.json @@ -0,0 +1,17 @@ +{ + "openapi": "3.0.0", + "paths": { + }, + "info": { + "title": "404", + "description": "", + "version": "", + "contact": {} + }, + "tags": [], + "servers": [], + "components": { + + "schemas": {} + } +} diff --git a/tsconfig.json b/tsconfig.json index 3923786..f20e56f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,6 +17,7 @@ "noImplicitAny": false, "strictBindCallApply": false, "forceConsistentCasingInFileNames": false, - "noFallthroughCasesInSwitch": false + "noFallthroughCasesInSwitch": false, + "resolveJsonModule": true } }