gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-11-04 21:29:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main

Browse Source
This commit is contained in:
Victor Radael
2022-07-07 16:36:28 -03:00
parent 6aa7be98d2
commit 6803ab3420
8 changed files with 42 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+8 -8
View File
@@ -35,8 +35,8 @@
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"protospack": "2.5.1",
"protospack-v2": "3.0.0",
"protospack": "2.5.2",
"protospack-v2": "3.0.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
@@ -8997,9 +8997,9 @@
}
},
"node_modules/protospack-v2": {
"version": "3.0.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/protospack-v2/-/protospack-v2-3.0.0.tgz",
"integrity": "sha512-GEFbf8+vVaq/lA28K4nFf7bOSlghQJn/aeVnrYzh4YJUaa1+ccNz3uQs9p5OGbfjt2u/fFE5rHGEBUtueAA8KA==",
"version": "3.0.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/protospack-v2/-/protospack-v2-3.0.1.tgz",
"integrity": "sha512-gbcuF4Q+n+nikKB8jjuM+MJ0PyUAX3puJsk2/u47FhRcwkFB8mjj1a1xjYsJeOBHqGVq88QDemcIwxGEEQpX0Q==",
"dependencies": {
"rxjs": "^7.5.5",
"ts-proto": "^1.112.2"
@@ -17198,9 +17198,9 @@
}
},
"protospack-v2": {
"version": "3.0.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/protospack-v2/-/protospack-v2-3.0.0.tgz",
"integrity": "sha512-GEFbf8+vVaq/lA28K4nFf7bOSlghQJn/aeVnrYzh4YJUaa1+ccNz3uQs9p5OGbfjt2u/fFE5rHGEBUtueAA8KA==",
"version": "3.0.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/protospack-v2/-/protospack-v2-3.0.1.tgz",
"integrity": "sha512-gbcuF4Q+n+nikKB8jjuM+MJ0PyUAX3puJsk2/u47FhRcwkFB8mjj1a1xjYsJeOBHqGVq88QDemcIwxGEEQpX0Q==",
"requires": {
"rxjs": "^7.5.5",
"ts-proto": "^1.112.2"
package.json
+2 -2
View File
@@ -50,8 +50,8 @@
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"protospack": "2.5.1",
"protospack-v2": "3.0.0",
"protospack": "2.5.2",
"protospack-v2": "3.0.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
src/app.module.ts
+3 -10
View File
@@ -37,15 +37,13 @@ import { APP_GUARD } from '@nestjs/core';
import { GoogleStrategy } from './modules/oauth/passport-strategies/google-strategy';
import { MailChimpStrategy } from './modules/oauth/passport-strategies/mailchimp-strategy';
import { SalesforceStrategy } from './modules/oauth/passport-strategies/salesforce-strategy';
import { ConnectorClientConfiguration } from './clients/connector/client.config';
import { ConnectorController } from './modules/connector/connector.controller';
import { ConnectorClientService } from './clients/connector/client.service';
import { ConnectorModule } from './modules/connector/connector.module';
const inputClient = new InputsClientConfiguration();
const outputClient = new OutputsClientConfiguration();
const pipelineClient = new PipelinesClientConfiguration();
const transformationClient = new TransformationsClientConfiguration();
const connectorClient = new ConnectorClientConfiguration();
@Module({
controllers: [
@@ -56,7 +54,6 @@ const connectorClient = new ConnectorClientConfiguration();
HealthController,
CatalogController,
OauthController,
ConnectorController,
],
providers: [
DadosferaLogger,
@@ -65,7 +62,6 @@ const connectorClient = new ConnectorClientConfiguration();
TransformationsService,
OutputsService,
PipelinesService,
ConnectorClientService,
HealthService,
InputsClientService,
TransformationsClientService,
@@ -88,6 +84,7 @@ const connectorClient = new ConnectorClientConfiguration();
}),
AuthModule,
PermissionsModule,
ConnectorModule,
ClientsModule.register([
{
@@ -106,10 +103,6 @@ const connectorClient = new ConnectorClientConfiguration();
name: 'PIPELINES_PACKAGE',
...pipelineClient.config(),
},
{
name: 'CONNECTOR_PACKAGE',
...connectorClient.config(),
},
]),
],
})
src/clients/connector/client.config.ts
-31
View File
@@ -1,31 +0,0 @@
import { credentials } from '@grpc/grpc-js';
import { ClientOptions, Transport } from '@nestjs/microservices';
import { ConnectorManager } from 'protospack-v2';
export class ConnectorClientConfiguration {
config(): ClientOptions {
return {
transport: Transport.GRPC,
options: {
url: process.env.INFACTORY_URL,
package: [
ConnectorManager.ProtoPackages.WritePackage,
ConnectorManager.ProtoPackages.ReadPackage,
],
credentials:
process.env.LOCAL_ENV === 'local'
? undefined
: credentials.createSsl(),
protoPath: [
ConnectorManager.ProtoPaths.WriteFilePath,
ConnectorManager.ProtoPaths.ReadFilePath,
],
loader: {
enums: String,
objects: true,
arrays: true,
},
},
};
}
}
src/clients/connector/client.service.ts
-334
View File
@@ -1,334 +0,0 @@
import {
OnModuleInit,
Inject,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { DadosferaLogger } from 'dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { ConnectorManager } from 'protospack-v2';
import { from } from 'rxjs';
export class ConnectorClientService implements OnModuleInit {
private connectorServiceRead: ConnectorManager.ReadService.ConnectorManagerReadServices;
private connectorServiceWrite: ConnectorManager.WriteService.ConnectorManagerWriteServices;
logger: any;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject('CONNECTOR_PACKAGE') private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.connectorServiceWrite =
this.grpcClient.getService<ConnectorManager.WriteService.ConnectorManagerWriteServices>(
ConnectorManager.ProtoServices.ConnectorManagerWriteServices,
);
this.connectorServiceRead =
this.grpcClient.getService<ConnectorManager.ReadService.ConnectorManagerReadServices>(
ConnectorManager.ProtoServices.ConnectorManagerReadServices,
);
}
async uploadConnector(uploadConnector) {
const connector: ConnectorManager.Entities.ConnectorCreateRequest = {
file: {
buffer: uploadConnector.file.buffer,
mimetypes: uploadConnector.file.mimetype,
},
connector: JSON.stringify(uploadConnector.connector),
};
const serviceBody: ConnectorManager.Messages.RegisterConnectorRequest = {
connector,
};
const observable = from(
this.connectorServiceWrite.RegisterConnector(serviceBody),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.details);
this.logger.error(err);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
this.logger.info('ConnectorClientService - Upload');
return result;
}
async uploadFile(uploadFile) {
const body: ConnectorManager.Messages.UploadFileRequest = {
file: {
buffer: uploadFile.file.buffer,
mimetypes: uploadFile.file.mimetype,
},
name: uploadFile.name,
};
const observable = from(this.connectorServiceWrite.UploadFile(body));
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err.details);
this.logger.error(err);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
this.logger.info('ConnectorClientService - Upload');
return result;
}
async getAllConnectors(body) {
const observable = from(
this.connectorServiceRead.GetAllConnectors({
search: body.search,
filters: JSON.stringify(body.filters),
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
this.logger.info('ConnectorClientService - getAllConnectors');
return result;
}
async getConnectorDetails(plugin: string) {
const observable = from(
this.connectorServiceRead.GetConnectorDetails({
plugin,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
this.logger.info('ConnectorClientService - getAllConnectors');
return result;
}
async getConnector(plugin: string) {
const observable = from(
this.connectorServiceRead.GetConnector({
plugin,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
this.logger.info('ConnectorClientService - getAllConnectors');
return result;
}
async getConnectorsTags() {
const observable = from(this.connectorServiceRead.GetConnectorsTags({}));
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
}).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
console.log('ConnectorClientService', 'getAllConnectors');
return result;
}
async deleteConnector(plugin: string, version: string) {
const observable = from(
this.connectorServiceWrite.DeleteConnector({
version,
plugin,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
this.logger.info('ConnectorClientService - getAllConnectors');
return result;
}
async updateConnector({ plugin, changes }) {
const observable = from(
this.connectorServiceWrite.UpdateConnector({
plugin,
changes,
}),
);
const result = await new Promise((resolve, reject) => {
observable.subscribe({
next(x) {
resolve(x);
},
error(err) {
reject(err);
},
complete() {
//console.log('done');
},
});
})
.then((res) => {
this.logger.info('Done');
return res;
})
.catch((err) => {
this.logger.error(err);
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 400,
);
});
this.logger.info('ConnectorClientService - getAllConnectors');
return result;
}
}
src/clients/duc/client.config.ts
+1 -4
View File
@@ -9,10 +9,7 @@ export class DucClient {
options: {
url: process.env.DUC_URL,
package: ProtoPackages.WritePackage,
credentials:
process.env.LOCAL_ENV === 'local'
? undefined
: credentials.createSsl(),
credentials: credentials.createSsl(),
protoPath: ProtoPaths.WriteFilePath,
loader: {
keepCase: true,
src/modules/connector/connector.controller.ts
+19 -2
View File
@@ -14,9 +14,10 @@ import {
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
import { ConnectorClientService } from 'src/clients/connector/client.service';
import { ConnectorClientService } from './client.service';
import { AddTagDto } from './dtos/add-tag';
import { CreateConnectorDto } from './dtos/create-connector';
import { UploadConnectorsFileDto } from './dtos/create-multiple-connector';
import { DeleteConnectorDto } from './dtos/delete-connector';
import { GetAllDto } from './dtos/get-all';
import { RemoveTagDto } from './dtos/remove-tag';
@@ -68,15 +69,31 @@ export class ConnectorController {
return response;
}
@Post('/multiple-connectors-upload')
@ApiConsumes('multipart/form-data')
@UseInterceptors(FileInterceptor('file'))
async uploadConnectorsFile(@UploadedFile() file: UploadConnectorsFileDto) {
console.log(`/upload`, 'Upload Connector Route');
const response = await this.connectorClientService.uploadConnectorsFile({
file,
});
return response;
}
@Get()
async getAllConnectors(@Query() queries: GetAllDto) {
this.logger.info('/upload - Upload Connector Route');
const { search, ...filters } = queries;
const { search, language, size, page, ...filters } = queries;
const response: any = await this.connectorClientService.getAllConnectors({
filters: filters || {},
search: search || '',
language: language || 'pt-br',
size: size || 10,
page: page || 1,
});
const connectors = JSON.parse(response.connectors).connectors;
src/modules/connector/dtos/get-all.ts
+9
View File
@@ -4,6 +4,15 @@ export class GetAllDto {
@ApiProperty()
search: string;
@ApiProperty()
language: string;
@ApiProperty()
size: string;
@ApiProperty()
page: string;
@ApiProperty()
filters: any;
}