Register Connector

This commit is contained in:
Victor Radael
2022-06-20 15:37:45 -03:00
parent 03ed4d88a0
commit 953c0ac41e
14 changed files with 845 additions and 84 deletions
+10
View File
@@ -31,12 +31,16 @@ import { InputsClientConfiguration } from './clients/inputs/client.config';
import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
import { CatalogController } from './modules/catalog/catalog.controller';
import { CatalogService } from './modules/catalog/catalog.service';
import { ConnectorClientConfiguration } from './clients/connector/client.config';
import { ConnectorController } from './modules/connector/connector.controller';
import { ConnectorClientService } from './clients/connector/client.service';
const authClient = new AuthClient();
const inputClient = new InputsClientConfiguration();
const outputClient = new OutputsClientConfiguration();
const pipelineClient = new PipelinesClientConfiguration();
const transformationClient = new TransformationsClientConfiguration();
const connectorClient = new ConnectorClientConfiguration();
@Module({
controllers: [
@@ -47,6 +51,7 @@ const transformationClient = new TransformationsClientConfiguration();
AuthController,
HealthController,
CatalogController,
ConnectorController,
],
providers: [
InputsService,
@@ -54,6 +59,7 @@ const transformationClient = new TransformationsClientConfiguration();
OutputsService,
PipelinesService,
// AuthService,
ConnectorClientService,
HealthService,
InputsClientService,
TransformationsClientService,
@@ -89,6 +95,10 @@ const transformationClient = new TransformationsClientConfiguration();
name: 'PIPELINES_PACKAGE',
...pipelineClient.config(),
},
{
name: 'CONNECTOR_PACKAGE',
...connectorClient.config(),
},
]),
],
})
+30
View File
@@ -0,0 +1,30 @@
import { credentials } from '@grpc/grpc-js';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { ConnectorManager } from '@victorradael/protospack-v2';
export class ConnectorClientConfiguration {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.INFACTORY_URL,
package: [
ConnectorManager.ProtoPackages.WritePackage,
ConnectorManager.ProtoPackages.ReadPackage,
],
credentials: process.env.LOCAL_ENV
? undefined
: credentials.createSsl(),
protoPath: [
ConnectorManager.ProtoPaths.WriteFilePath,
ConnectorManager.ProtoPaths.ReadFilePath,
],
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
+229
View File
@@ -0,0 +1,229 @@
import {
OnModuleInit,
Inject,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { ConnectorManager } from '@victorradael/protospack-v2';
import { from } from 'rxjs';
export class ConnectorClientService implements OnModuleInit {
private connectorServiceRead: ConnectorManager.ReadService.ConnectorManagerReadServices;
private connectorServiceWrite: ConnectorManager.WriteService.ConnectorManagerWriteServices;
constructor(
@Inject('CONNECTOR_PACKAGE') private readonly grpcClient: ClientGrpc,
) {}
onModuleInit() {
this.connectorServiceWrite =
this.grpcClient.getService<ConnectorManager.WriteService.ConnectorManagerWriteServices>(
ConnectorManager.ProtoServices.ConnectorManagerWriteServices,
);
this.connectorServiceRead =
this.grpcClient.getService<ConnectorManager.ReadService.ConnectorManagerReadServices>(
ConnectorManager.ProtoServices.ConnectorManagerReadServices,
);
}
async uploadConnector(uploadConnector) {
const connector: ConnectorManager.Entities.ConnectorCreateRequest = {
file: {
buffer: uploadConnector.file.buffer,
mimetypes: uploadConnector.file.mimetype,
},
connector: uploadConnector.connector,
};
const serviceBody: ConnectorManager.Messages.RegisterConnectorRequest = {
connectors: [connector],
};
const observable = from(
this.connectorServiceWrite.RegisterConnector(serviceBody),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
console.log('ConnectorClientService', 'Upload');
return result;
}
async getAllConnectors(
body: ConnectorManager.Messages.GetAllConnectorsRequest,
) {
const observable = from(
this.connectorServiceRead.GetAllConnectors({
search: body.search,
filters: JSON.stringify(body.filters),
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
console.log('ConnectorClientService', 'getAllConnectors');
return result;
}
async getConnectorDetails(plugin: string) {
const observable = from(
this.connectorServiceRead.GetConnectorDetails({
plugin,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
console.log('ConnectorClientService', 'getAllConnectors');
return result;
}
async getConnector(plugin: string) {
const observable = from(
this.connectorServiceRead.GetConnector({
plugin,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
console.log('ConnectorClientService', 'getAllConnectors');
return result;
}
async deleteConnector(plugin: string) {
const observable = from(
this.connectorServiceWrite.DeleteConnector({
plugin,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
console.log('ConnectorClientService', 'getAllConnectors');
return result;
}
async updateConnector({ plugin, changes }) {
const observable = from(
this.connectorServiceWrite.UpdateConnector({
plugin,
changes,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
console.log('ConnectorClientService', 'getAllConnectors');
return result;
}
}
+6 -2
View File
@@ -3,6 +3,8 @@ import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import { writeFileSync } from 'fs';
import helmet from 'helmet';
import documentEmpty from '../swagger_empty.json';
import { AppModule } from './app.module';
async function bootstrap() {
@@ -18,7 +20,7 @@ async function bootstrap() {
app.use(helmet());
const config = new DocumentBuilder()
.setTitle('Maestro Grpc Documentation')
.setTitle('Maestro')
.setDescription('Documentation for Maestro gateway')
.setVersion('1.0')
.addBearerAuth()
@@ -26,7 +28,9 @@ async function bootstrap() {
const document = SwaggerModule.createDocument(app, config);
writeFileSync('./swagger.json', JSON.stringify(document));
SwaggerModule.setup('api', app, document);
const swaggerDoc = process.env.ENV === 'dev' ? document : documentEmpty;
SwaggerModule.setup('api', app, swaggerDoc);
await app.listen(3333);
}
@@ -0,0 +1,180 @@
import {
Body,
Controller,
Delete,
Get,
HttpException,
Param,
Post,
Put,
UploadedFile,
UseInterceptors,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { ApiConsumes, ApiTags } from '@nestjs/swagger';
import { ConnectorManager } from '@victorradael/protospack-v2';
import { ConnectorClientService } from 'src/clients/connector/client.service';
import { AddTagDto } from './dtos/add-tag';
import { CreateConnectorDto } from './dtos/create-connector';
import { RemoveTagDto } from './dtos/remove-tag';
import { UpdateDto } from './dtos/update';
@ApiTags('connectors')
@Controller('connectors')
export class ConnectorController {
constructor(private connectorClientService: ConnectorClientService) {}
@Post('/')
@ApiConsumes('multipart/form-data')
@UseInterceptors(FileInterceptor('file'))
async uploadConnector(
@UploadedFile() file,
@Body() body: CreateConnectorDto,
) {
console.log(`/upload`, 'Upload Connector Route');
console.log(file, body);
// const response = await this.connectorClientService.uploadConnector({
// file: fileField.file,
// connector: body.connector,
// });
// console.log('X', response);
// return response;
}
@Get('/')
async getAllConnectors(
@Body() body: ConnectorManager.Messages.GetAllConnectorsRequest,
) {
console.log(`/upload`, 'Upload Connector Route');
const { filters, search } = body;
const response: any = await this.connectorClientService.getAllConnectors({
filters,
search,
});
return {
message: response.message,
connector: { ...JSON.parse(response.connector) },
};
}
@Get('/:plugin')
async getConnector(@Param('plugin') plugin: string) {
console.log(`/upload`, 'Upload Connector Route');
const response: any = await this.connectorClientService.getConnector(
plugin,
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Get('/:plugin/details')
async getConnectorDetails(@Param('plugin') plugin: string) {
console.log(`/upload`, 'Upload Connector Route');
const response: any = await this.connectorClientService.getConnectorDetails(
plugin,
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Put('/:plugin')
@ApiConsumes('multipart/form-data')
async updateConnector(
@Param('plugin') plugin: string,
@Body() body: UpdateDto,
) {
console.log(`/upload`, 'Upload Connector Route');
const changes = body;
const response: any = await this.connectorClientService.updateConnector({
plugin,
changes: JSON.stringify(changes),
});
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Put('/:plugin/add-tag')
async addTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: AddTagDto,
) {
console.log(`/upload`, 'Upload Connector Route');
const { tags } = body;
if (!tags || !(typeof tags === 'object') || !tags.length) {
throw new HttpException('Not found tags attributes', 400);
}
const changes = { plugin, tags };
const response: any = await this.connectorClientService.updateConnector({
plugin,
changes: JSON.stringify(changes),
});
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Put('/:plugin/remove-tag')
async removeTagOnConnector(
@Param('plugin') plugin: string,
@Body() body: RemoveTagDto,
) {
console.log(`/upload`, 'Upload Connector Route');
const { tags } = body;
const remove = tags;
if (!remove || !(typeof remove === 'object') || !remove.length) {
throw new HttpException('Not found remove attribute', 400);
}
const changes = { plugin, remove };
const response: any = await this.connectorClientService.updateConnector({
plugin,
changes: JSON.stringify(changes),
});
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
@Delete('/:plugin')
async deleteConnector(@Param('plugin') plugin: string) {
console.log(`/upload`, 'Upload Connector Route');
const response: any = await this.connectorClientService.deleteConnector(
plugin,
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
}
}
+6
View File
@@ -0,0 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
export class AddTagDto {
@ApiProperty()
tags: string[];
}
@@ -0,0 +1,37 @@
import { ApiProperty } from '@nestjs/swagger';
export class CreateConnectorDto {
@ApiProperty({ format: 'binary' })
file: string;
@ApiProperty()
category: string;
@ApiProperty()
plugin: string;
@ApiProperty()
image: string;
@ApiProperty()
docs: string;
@ApiProperty()
name: string;
@ApiProperty({
description: 'skip_select_entities: boolean skip_select_columns: boolean;',
})
options: {
skip_select_entities: boolean;
skip_select_columns: boolean;
};
@ApiProperty({
type: 'array',
items: {
properties: {},
},
})
connection_controls: object[];
}
+6
View File
@@ -0,0 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
export class RemoveTagDto {
@ApiProperty()
tags: string[];
}
+39
View File
@@ -0,0 +1,39 @@
import { ApiProperty } from '@nestjs/swagger';
export class UpdateDto {
@ApiProperty({ format: 'binary', required: false })
file?: string;
@ApiProperty({ required: false })
category?: string;
@ApiProperty({ required: false })
plugin?: string;
@ApiProperty({ required: false })
image?: string;
@ApiProperty({ required: false })
docs?: string;
@ApiProperty({ required: false })
name?: string;
@ApiProperty({
required: false,
description: 'skip_select_entities: boolean skip_select_columns: boolean;',
})
options?: {
skip_select_entities: boolean;
skip_select_columns: boolean;
};
@ApiProperty({
required: false,
type: 'array',
items: {
properties: {},
},
})
connection_controls?: object[];
}