diff --git a/src/app.module.ts b/src/app.module.ts index e0649c2..3b3c50c 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -4,11 +4,12 @@ import { UsersGrpcClientModule } from './users-grpc-client/users-grpc-client.mod import { OutputsModule } from './outputs/outputs.module'; import { EnrichmentModule } from './enrichment/enrichment.module'; import { TransformationsModule } from './transformations/transformations.module'; +import { PipelinesModule } from './pipelines/pipelines.module'; @Module({ imports: [ InputGrpcClientModule, - OutputsModule, EnrichmentModule, TransformationsModule + OutputsModule, EnrichmentModule, TransformationsModule, PipelinesModule ], controllers: [], providers: [], diff --git a/src/pipelines/config/grpc-client.ts b/src/pipelines/config/grpc-client.ts new file mode 100644 index 0000000..af87bbe --- /dev/null +++ b/src/pipelines/config/grpc-client.ts @@ -0,0 +1,17 @@ +import { join } from 'path'; +import { ClientOptions, Transport } from '@nestjs/microservices'; + +export const GrpcClientConfiguration: ClientOptions = { + transport: Transport.GRPC, + options: { + //url: `${process.env.USERS_SVC_URL}:${process.env.USERS_SVC_PORT}`, + url: `127.0.0.1:50054`, + package: 'pipeline', + protoPath: join(__dirname, '..', 'proto', 'pipeline.proto'), + loader: { + enums: String, + objects: true, + arrays: true, + }, + }, +}; diff --git a/src/pipelines/dto/create-pipeline.dto.ts b/src/pipelines/dto/create-pipeline.dto.ts new file mode 100644 index 0000000..cb543d8 --- /dev/null +++ b/src/pipelines/dto/create-pipeline.dto.ts @@ -0,0 +1,12 @@ +import { InputGrpcClient } from "src/inputs/entities/input-grcp-client.entity"; +import { Output } from "src/outputs/entities/output.entity"; +import { Transformation } from "src/transformations/entities/transformation.entity"; + +export class CreatePipelineDto { + input:InputGrpcClient; + output:Output; + transformations:Transformation; + tags:string[]; + name:string; + description:string; +} diff --git a/src/pipelines/dto/update-pipeline.dto.ts b/src/pipelines/dto/update-pipeline.dto.ts new file mode 100644 index 0000000..b1848b4 --- /dev/null +++ b/src/pipelines/dto/update-pipeline.dto.ts @@ -0,0 +1,6 @@ +import { PartialType } from '@nestjs/mapped-types'; +import { CreatePipelineDto } from './create-pipeline.dto'; + +export class UpdatePipelineDto extends PartialType(CreatePipelineDto) { + +} diff --git a/src/pipelines/entities/pipeline.entity.ts b/src/pipelines/entities/pipeline.entity.ts new file mode 100644 index 0000000..2e4eafb --- /dev/null +++ b/src/pipelines/entities/pipeline.entity.ts @@ -0,0 +1,14 @@ +import { InputGrpcClient } from "src/inputs/entities/input-grcp-client.entity"; +import { Output } from "src/outputs/entities/output.entity"; +import { Transformation } from "src/transformations/entities/transformation.entity"; + +export class Pipeline { + id:string; + input:InputGrpcClient; + output:Output; + transformations:Transformation; + tags:string[]; + name:string; + description:string; + status:string; +} diff --git a/src/pipelines/pipelines.controller.spec.ts b/src/pipelines/pipelines.controller.spec.ts new file mode 100644 index 0000000..13e92e9 --- /dev/null +++ b/src/pipelines/pipelines.controller.spec.ts @@ -0,0 +1,20 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PipelinesController } from './pipelines.controller'; +import { PipelineGrpcClientService } from './pipelines.service'; + +describe('PipelinesController', () => { + let controller: PipelinesController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [PipelinesController], + providers: [PipelineGrpcClientService], + }).compile(); + + controller = module.get(PipelinesController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/src/pipelines/pipelines.controller.ts b/src/pipelines/pipelines.controller.ts new file mode 100644 index 0000000..b624cf1 --- /dev/null +++ b/src/pipelines/pipelines.controller.ts @@ -0,0 +1,56 @@ +import { Controller, Delete, Get, OnModuleInit, Param, Post, Put } from '@nestjs/common'; +import { Client, ClientGrpc, MessagePattern, Payload } from '@nestjs/microservices'; +import { PipelineGrpcClientService } from './pipelines.service'; +import { CreatePipelineDto } from './dto/create-pipeline.dto'; +import { UpdatePipelineDto } from './dto/update-pipeline.dto'; +import { GrpcClientConfiguration } from './config/grpc-client'; +import { ApiTags } from '@nestjs/swagger'; + +@ApiTags('pipelines') +@Controller('pipeline') +export class PipelinesController implements OnModuleInit{ + + @Client(GrpcClientConfiguration) + private pipelineServiceClient: ClientGrpc; + + private pipelineService: PipelineGrpcClientService; + + onModuleInit() { + this.pipelineService = + this.pipelineServiceClient.getService('PipelineService'); + } + + @Post() + create(@Payload() createPipelineDto: CreatePipelineDto) { + return this.pipelineService.create(createPipelineDto); + } + + @Get() + list() { + return this.pipelineService.list(); + } + + @Get(':id') + show(@Param() id: string) { + return this.pipelineService.show(id); + } + + @Put(':id') + update(@Param() id:string,@Payload() updatePipelineDto: UpdatePipelineDto) { + return this.pipelineService.update(id, updatePipelineDto); + } + + @Delete() + remove(@Param() id:string) { + return this.pipelineService.remove(id); + } + + @Post() + getPipelineLogsMessages(@Param() id:string,@Payload() details:string){ + return this.pipelineService.getPipelineLogsMessages(id,details); + } + + getPipelineStatus(id:string){ + return this.pipelineService.getPipelineStatus(id) + } +} diff --git a/src/pipelines/pipelines.module.ts b/src/pipelines/pipelines.module.ts new file mode 100644 index 0000000..ed8c320 --- /dev/null +++ b/src/pipelines/pipelines.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { PipelineGrpcClientService } from './pipelines.service'; +import { PipelinesController } from './pipelines.controller'; + +@Module({ + controllers: [PipelinesController], + providers: [PipelineGrpcClientService] +}) +export class PipelinesModule {} diff --git a/src/pipelines/pipelines.service.spec.ts b/src/pipelines/pipelines.service.spec.ts new file mode 100644 index 0000000..e7a30b6 --- /dev/null +++ b/src/pipelines/pipelines.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PipelinesService } from './pipelines.service'; + +describe('PipelinesService', () => { + let service: PipelinesService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [PipelinesService], + }).compile(); + + service = module.get(PipelinesService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/pipelines/pipelines.service.ts b/src/pipelines/pipelines.service.ts new file mode 100644 index 0000000..bcfede5 --- /dev/null +++ b/src/pipelines/pipelines.service.ts @@ -0,0 +1,34 @@ +import { Injectable } from '@nestjs/common'; +import { CreatePipelineDto } from './dto/create-pipeline.dto'; +import { UpdatePipelineDto } from './dto/update-pipeline.dto'; + +@Injectable() +export class PipelineGrpcClientService { + create(createPipelineDto: CreatePipelineDto) { + return 'This action adds a new pipeline'; + } + + list() { + return `This action returns all pipelines`; + } + + show(id: string) { + return `This action returns a #${id} pipeline`; + } + + update(id: string, updatePipelineDto: UpdatePipelineDto) { + return `This action updates a #${id} pipeline`; + } + + remove(id: string) { + return `This action removes a #${id} pipeline`; + } + + getPipelineLogsMessages(id:string,details:string){ + return `This action perfoms a getLogMessages on #${id} pipeline`; + } + + getPipelineStatus(id:string){ + return `This action performs a getStatus on #${id} pipeline`; + } +} diff --git a/src/pipelines/proto/pipeline.proto b/src/pipelines/proto/pipeline.proto new file mode 100644 index 0000000..58de33f --- /dev/null +++ b/src/pipelines/proto/pipeline.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package pipeline; + +import "google/protobuf/timestamp.proto"; + +service PipelineService{ + rpc Create(CreatePipelineRequest) returns (Pipeline) {} + rpc Show(IdRequest) returns (Pipeline) {} + rpc List(Empty) returns (ListResponse){} + rpc Update(UpdatePipelineRequest) returns (Pipeline){} + rpc Remove(IdRequest) returns (Empty){} + rpc getPipelineLogsMessages(getPipelineLogsMessagesRequest) returns (getPipelineLogsMessagesResponse){} + rpc getPipelineStatus(IdRequest) returns (getPipelineStatusResponse){} +} + + +message Empty {} + +message IdRequest{ + string id = 1; +} + +message Transformation { + string id = 1; + string inputSource = 2; + string table = 3; + string type = 4; + string params = 5; +} + +message Input { + string id = 1; + string cronFormated = 2; + string name = 3; + string plugin = 4; + string values = 5; + string operation = 6; +} + +message Output { + string id = 1; + string name = 2; + string plugin = 3; + string values = 4; + string operation = 5; +} + +message Pipeline { + string id = 1; + Input input = 2; + Output output = 3; + Transformation transformations = 4; + repeated string tags = 5; + string name = 6; + string description = 7; + string status = 8; +} + +message CreatePipelineRequest { + Input input = 1; + Output output = 2; + Transformation transformations = 3; + repeated string tags = 4; + string name = 5; + string description = 6; +} + +message UpdatePipelineRequest{ + string id = 1; + repeated string tags = 2; + string name = 3; + string description = 4; + string status = 5; +} + +message ListResponse { + repeated Pipeline pipelines = 1; +} + +message getPipelineLogsMessagesRequest{ + string id = 1; + string details = 2; +} + +message getPipelineLogsMessagesResponseItem{ + string date = 1; + string log = 2; +} + +message getPipelineLogsMessagesResponse{ + repeated getPipelineLogsMessagesResponseItem pipelineLogs = 1; +} + +message getPipelineStatusResponse{ + string status = 1; +} \ No newline at end of file diff --git a/swagger.json b/swagger.json index 63768f9..3a1d124 100644 --- a/swagger.json +++ b/swagger.json @@ -1 +1 @@ -{"openapi":"3.0.0","paths":{"/input":{"post":{"operationId":"InputGrpcClientController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInputGrpcClientDto"}}}},"responses":{"201":{"description":""}},"tags":["inputs"]},"get":{"operationId":"InputGrpcClientController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]}},"/input/{id}":{"get":{"operationId":"InputGrpcClientController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]},"put":{"operationId":"InputGrpcClientController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInputGrpcClientDto"}}}},"responses":{"200":{"description":""}},"tags":["inputs"]},"delete":{"operationId":"InputGrpcClientController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]}},"/input/test-connection":{"get":{"operationId":"InputGrpcClientController_testConnection","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestConnectionDTO"}}}},"responses":{"200":{"description":""}},"tags":["inputs"]}},"/output":{"post":{"operationId":"OutputsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateOutputDto"}}}},"responses":{"201":{"description":""}},"tags":["outputs"]},"get":{"operationId":"OutputsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["outputs"]}},"/output/{id}":{"get":{"operationId":"OutputsController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["outputs"]},"put":{"operationId":"OutputsController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateOutputDto"}}}},"responses":{"200":{"description":""}},"tags":["outputs"]},"delete":{"operationId":"OutputsController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["outputs"]}},"/enrichment":{"post":{"operationId":"EnrichmentController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateEnrichmentDto"}}}},"responses":{"201":{"description":""}},"tags":["enrichments"]},"get":{"operationId":"EnrichmentController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["enrichments"]}},"/enrichment/{id}":{"get":{"operationId":"EnrichmentController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["enrichments"]},"put":{"operationId":"EnrichmentController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateEnrichmentDto"}}}},"responses":{"200":{"description":""}},"tags":["enrichments"]},"delete":{"operationId":"EnrichmentController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["enrichments"]}},"/transformation":{"post":{"operationId":"TransformationsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"array","items":{"type":"string"}}}}},"responses":{"201":{"description":""}},"tags":["transformations"]},"get":{"operationId":"TransformationsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["transformations"]}},"/transformation/{id}":{"get":{"operationId":"TransformationsController_show","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"string"}}}},"responses":{"200":{"description":""}},"tags":["transformations"]},"put":{"operationId":"TransformationsController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateTransformationDto"}}}},"responses":{"200":{"description":""}},"tags":["transformations"]},"delete":{"operationId":"TransformationsController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["transformations"]}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[{"name":"inputs","description":""}],"servers":[],"components":{"schemas":{"CreateInputGrpcClientDto":{"type":"object","properties":{"cron":{"type":"string"},"name":{"type":"string","description":"Name of the input","example":"Test Name"},"plugin":{"type":"string"},"values":{"type":"string"},"operation":{"type":"string"}},"required":["cron","name","plugin","values","operation"]},"UpdateInputGrpcClientDto":{"type":"object","properties":{}},"TestConnectionDTO":{"type":"object","properties":{}},"CreateOutputDto":{"type":"object","properties":{}},"UpdateOutputDto":{"type":"object","properties":{}},"CreateEnrichmentDto":{"type":"object","properties":{}},"UpdateEnrichmentDto":{"type":"object","properties":{}},"UpdateTransformationDto":{"type":"object","properties":{}}}}} \ No newline at end of file +{"openapi":"3.0.0","paths":{"/input":{"post":{"operationId":"InputGrpcClientController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateInputGrpcClientDto"}}}},"responses":{"201":{"description":""}},"tags":["inputs"]},"get":{"operationId":"InputGrpcClientController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]}},"/input/{id}":{"get":{"operationId":"InputGrpcClientController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]},"put":{"operationId":"InputGrpcClientController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateInputGrpcClientDto"}}}},"responses":{"200":{"description":""}},"tags":["inputs"]},"delete":{"operationId":"InputGrpcClientController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["inputs"]}},"/input/test-connection":{"get":{"operationId":"InputGrpcClientController_testConnection","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/TestConnectionDTO"}}}},"responses":{"200":{"description":""}},"tags":["inputs"]}},"/output":{"post":{"operationId":"OutputsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateOutputDto"}}}},"responses":{"201":{"description":""}},"tags":["outputs"]},"get":{"operationId":"OutputsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["outputs"]}},"/output/{id}":{"get":{"operationId":"OutputsController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["outputs"]},"put":{"operationId":"OutputsController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateOutputDto"}}}},"responses":{"200":{"description":""}},"tags":["outputs"]},"delete":{"operationId":"OutputsController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["outputs"]}},"/enrichment":{"post":{"operationId":"EnrichmentController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateEnrichmentDto"}}}},"responses":{"201":{"description":""}},"tags":["enrichments"]},"get":{"operationId":"EnrichmentController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["enrichments"]}},"/enrichment/{id}":{"get":{"operationId":"EnrichmentController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["enrichments"]},"put":{"operationId":"EnrichmentController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateEnrichmentDto"}}}},"responses":{"200":{"description":""}},"tags":["enrichments"]},"delete":{"operationId":"EnrichmentController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["enrichments"]}},"/transformation":{"post":{"operationId":"TransformationsController_create","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"array","items":{"type":"string"}}}}},"responses":{"201":{"description":""}},"tags":["transformations"]},"get":{"operationId":"TransformationsController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["transformations"]}},"/transformation/{id}":{"get":{"operationId":"TransformationsController_show","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"string"}}}},"responses":{"200":{"description":""}},"tags":["transformations"]},"put":{"operationId":"TransformationsController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdateTransformationDto"}}}},"responses":{"200":{"description":""}},"tags":["transformations"]},"delete":{"operationId":"TransformationsController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["transformations"]}},"/pipeline":{"post":{"operationId":"PipelinesController_getPipelineLogsMessages","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"string"}}}},"responses":{"201":{"description":""}},"tags":["pipelines"]},"get":{"operationId":"PipelinesController_list","parameters":[],"responses":{"200":{"description":""}},"tags":["pipelines"]},"delete":{"operationId":"PipelinesController_remove","parameters":[],"responses":{"200":{"description":""}},"tags":["pipelines"]}},"/pipeline/{id}":{"get":{"operationId":"PipelinesController_show","parameters":[],"responses":{"200":{"description":""}},"tags":["pipelines"]},"put":{"operationId":"PipelinesController_update","parameters":[],"requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/UpdatePipelineDto"}}}},"responses":{"200":{"description":""}},"tags":["pipelines"]}}},"info":{"title":"Maestro Grpc Documentation","description":"Documentation for Maestro gateway","version":"1.0","contact":{}},"tags":[{"name":"inputs","description":""}],"servers":[],"components":{"schemas":{"CreateInputGrpcClientDto":{"type":"object","properties":{"cron":{"type":"string"},"name":{"type":"string","description":"Name of the input","example":"Test Name"},"plugin":{"type":"string"},"values":{"type":"string"},"operation":{"type":"string"}},"required":["cron","name","plugin","values","operation"]},"UpdateInputGrpcClientDto":{"type":"object","properties":{}},"TestConnectionDTO":{"type":"object","properties":{}},"CreateOutputDto":{"type":"object","properties":{}},"UpdateOutputDto":{"type":"object","properties":{}},"CreateEnrichmentDto":{"type":"object","properties":{}},"UpdateEnrichmentDto":{"type":"object","properties":{}},"UpdateTransformationDto":{"type":"object","properties":{}},"CreatePipelineDto":{"type":"object","properties":{}},"UpdatePipelineDto":{"type":"object","properties":{}}}}} \ No newline at end of file