mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-21 14:59:08 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
db2c7d6c02 | ||
|
|
0a99ce1aa4 | ||
|
|
39c66f030a | ||
|
|
1223e21ac4 |
@@ -71,11 +71,6 @@ jobs:
|
|||||||
sudo mv helmfile /usr/local/bin/
|
sudo mv helmfile /usr/local/bin/
|
||||||
helmfile --version
|
helmfile --version
|
||||||
|
|
||||||
- name: Install Helm Diff plugin
|
|
||||||
run: |
|
|
||||||
helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
|
|
||||||
helm diff version
|
|
||||||
|
|
||||||
- name: Debug Helm env
|
- name: Debug Helm env
|
||||||
run: |
|
run: |
|
||||||
helm env
|
helm env
|
||||||
@@ -107,5 +102,4 @@ jobs:
|
|||||||
- name: Run Helmfile Diff
|
- name: Run Helmfile Diff
|
||||||
env:
|
env:
|
||||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||||
HELM_PLUGINS: /home/runner/.local/share/helm/plugins
|
|
||||||
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
|
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
|
||||||
|
|||||||
+90
-45
@@ -3343,7 +3343,14 @@
|
|||||||
],
|
],
|
||||||
"responses": {
|
"responses": {
|
||||||
"200": {
|
"200": {
|
||||||
"description": ""
|
"description": "",
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"tags": [
|
"tags": [
|
||||||
@@ -3867,6 +3874,88 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/pipelines/start/{id}": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "PipelinesController_activate",
|
||||||
|
"summary": "",
|
||||||
|
"deprecated": true,
|
||||||
|
"description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"name": "id",
|
||||||
|
"required": true,
|
||||||
|
"in": "path",
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"201": {
|
||||||
|
"description": "",
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tags": [
|
||||||
|
"Pipelines"
|
||||||
|
],
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"access-token": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"access-token": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/pipelines/{id}/status": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "PipelinesController_getPipelineStatus",
|
||||||
|
"summary": "",
|
||||||
|
"deprecated": true,
|
||||||
|
"description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"name": "id",
|
||||||
|
"required": true,
|
||||||
|
"in": "path",
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": "",
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tags": [
|
||||||
|
"Pipelines"
|
||||||
|
],
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"access-token": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"access-token": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/transformations": {
|
"/transformations": {
|
||||||
"post": {
|
"post": {
|
||||||
"operationId": "TransformationsController_create",
|
"operationId": "TransformationsController_create",
|
||||||
@@ -4570,50 +4659,6 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": {
|
|
||||||
"get": {
|
|
||||||
"operationId": "PlatformApiController_getPipelineRunJobs",
|
|
||||||
"summary": "Get pipeline run jobs",
|
|
||||||
"parameters": [
|
|
||||||
{
|
|
||||||
"name": "pipelineId",
|
|
||||||
"required": true,
|
|
||||||
"in": "path",
|
|
||||||
"schema": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "runId",
|
|
||||||
"required": true,
|
|
||||||
"in": "path",
|
|
||||||
"schema": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "",
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"type": "object"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"tags": [
|
|
||||||
"Platform API"
|
|
||||||
],
|
|
||||||
"security": [
|
|
||||||
{
|
|
||||||
"access-token": []
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/platform/jobs/{jobId}/input": {
|
"/platform/jobs/{jobId}/input": {
|
||||||
"put": {
|
"put": {
|
||||||
"operationId": "PlatformApiController_updateJobInput",
|
"operationId": "PlatformApiController_updateJobInput",
|
||||||
|
|||||||
Generated
+14
-4
@@ -16,7 +16,8 @@
|
|||||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack-v2": "3.40.0-beta.10",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
|
"@dadosfera/protospack-v2": "3.40.0-beta.9",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
@@ -1734,10 +1735,19 @@
|
|||||||
"winston-log2gelf": "^2.4.0"
|
"winston-log2gelf": "^2.4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@dadosfera/protospack": {
|
||||||
|
"version": "2.5.3",
|
||||||
|
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
|
||||||
|
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"rxjs": "^7.5.5"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@dadosfera/protospack-v2": {
|
"node_modules/@dadosfera/protospack-v2": {
|
||||||
"version": "3.40.0-beta.10",
|
"version": "3.40.0-beta.9",
|
||||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
|
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.9.tgz",
|
||||||
"integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
|
"integrity": "sha512-8jbCpzxQnDax41yhw8yQJLB1VVp4PY8grj2H5oBgV+BCUf2y5lp0EPzLwVZsnOsfsXNAXHyNmbZbAjmJC1ipaw==",
|
||||||
"license": "ISC",
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
|
|||||||
+2
-1
@@ -34,7 +34,8 @@
|
|||||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack-v2": "3.40.0-beta.10",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
|
"@dadosfera/protospack-v2": "3.40.0-beta.9",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
|
|||||||
import { NetworkConfigModule } from './modules/network-config/network-config.module';
|
import { NetworkConfigModule } from './modules/network-config/network-config.module';
|
||||||
import { InputsModule } from './modules/inputs/inputs.module';
|
import { InputsModule } from './modules/inputs/inputs.module';
|
||||||
import { OauthModule } from './modules/oauth/oauth.module';
|
import { OauthModule } from './modules/oauth/oauth.module';
|
||||||
|
import { PipelinesModule } from './modules/pipelines/pipelines.module';
|
||||||
import { TransformationsModule } from './modules/transformations/transformations.module';
|
import { TransformationsModule } from './modules/transformations/transformations.module';
|
||||||
import { HealthModule } from './modules/health/health.module';
|
import { HealthModule } from './modules/health/health.module';
|
||||||
import { CatalogModule } from './modules/catalog/catalog.module';
|
import { CatalogModule } from './modules/catalog/catalog.module';
|
||||||
@@ -59,6 +60,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
|
|||||||
PermissionsModule,
|
PermissionsModule,
|
||||||
TermsOfUseModule,
|
TermsOfUseModule,
|
||||||
ConnectionTestModule,
|
ConnectionTestModule,
|
||||||
|
PipelinesModule,
|
||||||
TransformationsModule,
|
TransformationsModule,
|
||||||
UsersModule,
|
UsersModule,
|
||||||
RolesModule,
|
RolesModule,
|
||||||
|
|||||||
@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|||||||
import { CatalogController } from './catalog.controller';
|
import { CatalogController } from './catalog.controller';
|
||||||
import { CatalogClientConfiguration } from './catalog-client';
|
import { CatalogClientConfiguration } from './catalog-client';
|
||||||
import { ClientsModule } from '@nestjs/microservices';
|
import { ClientsModule } from '@nestjs/microservices';
|
||||||
|
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||||
import { UsersModule } from '../users/users.module';
|
import { UsersModule } from '../users/users.module';
|
||||||
import { RolesModule } from '../roles/roles.module';
|
import { RolesModule } from '../roles/roles.module';
|
||||||
import { CustomersModule } from '../customers/customers.module';
|
import { CustomersModule } from '../customers/customers.module';
|
||||||
import { ShareModule } from './share/share.module';
|
import { ShareModule } from './share/share.module';
|
||||||
import { CatalogService } from './catalog.service';
|
import { CatalogService } from './catalog.service';
|
||||||
|
import { MixpanelModule } from '../mixpanel/mixpanel.module';
|
||||||
|
|
||||||
const client = new CatalogClientConfiguration();
|
const client = new CatalogClientConfiguration();
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
ClientsModule.register([client.providerOptions]),
|
ClientsModule.register([client.providerOptions]),
|
||||||
|
OldPipelineModule,
|
||||||
UsersModule,
|
UsersModule,
|
||||||
RolesModule,
|
RolesModule,
|
||||||
CustomersModule,
|
CustomersModule,
|
||||||
|
|||||||
@@ -1,8 +1,4 @@
|
|||||||
export interface Info {
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
user_id: string;
|
|
||||||
customer_id: string;
|
|
||||||
customer: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Values {
|
interface Values {
|
||||||
jdbc_user: string;
|
jdbc_user: string;
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
|
||||||
|
import { ClientGrpc } from '@nestjs/microservices';
|
||||||
|
import {
|
||||||
|
PipelineServicesNames,
|
||||||
|
PipelinesServiceInterface,
|
||||||
|
} from '@dadosfera/protospack';
|
||||||
|
import { lastValueFrom } from 'rxjs';
|
||||||
|
|
||||||
|
import { IIdRequest } from './interfaces';
|
||||||
|
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
|
|
||||||
|
export class PipelinesClientService implements OnModuleInit {
|
||||||
|
private pipelineService: PipelinesServiceInterface;
|
||||||
|
logger: DadosferaLogger;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
@Inject(DadosferaLogger)
|
||||||
|
dadosferaLogger: DadosferaLogger,
|
||||||
|
@Inject(PipelinesClientConfiguration.name)
|
||||||
|
private readonly grpcClient: ClientGrpc,
|
||||||
|
) {
|
||||||
|
this.logger = dadosferaLogger.logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
onModuleInit() {
|
||||||
|
this.pipelineService =
|
||||||
|
this.grpcClient.getService<PipelinesServiceInterface>(
|
||||||
|
PipelineServicesNames.PipelineService,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getPipelineStatus(data) {
|
||||||
|
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
||||||
|
|
||||||
|
const statusPipelineResponse = await lastValueFrom(
|
||||||
|
this.pipelineService.getPipelineStatus(data),
|
||||||
|
)
|
||||||
|
.then((res) => {
|
||||||
|
const statusArray =
|
||||||
|
res.status?.sort((a, b) => {
|
||||||
|
if (a.id < b.id) {
|
||||||
|
return 1;
|
||||||
|
} else {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}) || [];
|
||||||
|
return { status: statusArray };
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
this.logger.error(err.message);
|
||||||
|
throw new Error(err);
|
||||||
|
});
|
||||||
|
this.logger.info('Done');
|
||||||
|
|
||||||
|
return statusPipelineResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
async runPipeline({ id, info }: IIdRequest) {
|
||||||
|
this.logger.info('PipelinesClientService - RunPipeline');
|
||||||
|
const statusPipelineResponse = await lastValueFrom(
|
||||||
|
this.pipelineService.triggerPipeline({ id, info }),
|
||||||
|
).catch((err) => {
|
||||||
|
this.logger.error(err.message);
|
||||||
|
throw new Error(err);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (statusPipelineResponse.status == false) {
|
||||||
|
throw new ConflictException(
|
||||||
|
'This pipeline is not ready yet to execute, Try again later!',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info('Done');
|
||||||
|
return statusPipelineResponse;
|
||||||
|
}
|
||||||
|
}
|
||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
|
|
||||||
|
export interface ICreatePipelineDto {
|
||||||
|
input: IdRequest;
|
||||||
|
transformations: IdRequest[];
|
||||||
|
output: IdRequest;
|
||||||
|
tags: string[];
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
info: Info;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IdRequest {
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IIdRequest {
|
||||||
|
id: string;
|
||||||
|
info: Info;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IUpdatePipelineRequest {
|
||||||
|
input: IdRequest;
|
||||||
|
transformations: IdRequest[];
|
||||||
|
output: IdRequest;
|
||||||
|
tags: string[];
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
id: string;
|
||||||
|
info: Info;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IGetPipelineLogsRequest {
|
||||||
|
id: string;
|
||||||
|
details: string;
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import {
|
||||||
|
ClientsProviderAsyncOptions,
|
||||||
|
GrpcOptions,
|
||||||
|
Transport,
|
||||||
|
} from '@nestjs/microservices';
|
||||||
|
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
|
||||||
|
import { credentials } from '@grpc/grpc-js';
|
||||||
|
|
||||||
|
const isLocalConnection =
|
||||||
|
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
||||||
|
process.env.PIFACTORY_URL.includes('0.0.0.0');
|
||||||
|
|
||||||
|
export class PipelinesClientConfiguration {
|
||||||
|
public name = 'PipelinesClientConfiguration';
|
||||||
|
private config: GrpcOptions = {
|
||||||
|
transport: Transport.GRPC,
|
||||||
|
options: {
|
||||||
|
url: process.env.PIFACTORY_URL,
|
||||||
|
package: PipelinePackages,
|
||||||
|
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
||||||
|
protoPath: PipelineProtoFilePath,
|
||||||
|
loader: {
|
||||||
|
keepCase: true,
|
||||||
|
enums: String,
|
||||||
|
defaults: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
providerOptions: ClientsProviderAsyncOptions = {
|
||||||
|
name: this.name,
|
||||||
|
...this.config,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
|
||||||
|
import { ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||||
|
import {
|
||||||
|
AuthenticateCondition,
|
||||||
|
Authenticated,
|
||||||
|
RequireSomePermission,
|
||||||
|
} from 'src/decorators/authentication.decorator';
|
||||||
|
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||||
|
import { PipelinesService } from './pipelines.service';
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||||
|
|
||||||
|
@ApiInternalOnlyController()
|
||||||
|
@ApiTags('Pipelines')
|
||||||
|
@Controller('pipelines')
|
||||||
|
@Authenticated()
|
||||||
|
export class PipelinesController {
|
||||||
|
logger: DadosferaLogger;
|
||||||
|
constructor(
|
||||||
|
@Inject(DadosferaLogger)
|
||||||
|
dadosferaLogger: DadosferaLogger,
|
||||||
|
private pipelineService: PipelinesService,
|
||||||
|
) {
|
||||||
|
this.logger = dadosferaLogger.logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Post('start/:id')
|
||||||
|
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||||
|
@ApiOperation({
|
||||||
|
deprecated: true,
|
||||||
|
description:
|
||||||
|
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
|
||||||
|
})
|
||||||
|
async activate(@Param('id') id: string, @Body() body) {
|
||||||
|
const { info } = body;
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
|
||||||
|
{
|
||||||
|
user: body.info.user_id,
|
||||||
|
customer: body.info.customer,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const response = await this.pipelineService.runPipeline({ id, info });
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get(':id/status')
|
||||||
|
@ApiOperation({
|
||||||
|
deprecated: true,
|
||||||
|
description:
|
||||||
|
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
|
||||||
|
})
|
||||||
|
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||||
|
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||||
|
body.id = id;
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
|
||||||
|
{
|
||||||
|
user: body.info.user_id,
|
||||||
|
customer: body.info.customer,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const response = await this.pipelineService.getPipelineStatus(body);
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { ClientsModule } from '@nestjs/microservices';
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
|
import { PipelinesController } from './pipelines.controller';
|
||||||
|
import { PipelinesService } from './pipelines.service';
|
||||||
|
|
||||||
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
|
import { PipelinesClientService } from './client.service';
|
||||||
|
|
||||||
|
const client = new PipelinesClientConfiguration();
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [ClientsModule.register([client.providerOptions])],
|
||||||
|
controllers: [PipelinesController],
|
||||||
|
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
|
||||||
|
exports: [PipelinesService],
|
||||||
|
})
|
||||||
|
export class PipelinesModule {}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
|
||||||
|
import { PipelinesClientService } from './client.service';
|
||||||
|
import { IIdRequest } from './interfaces';
|
||||||
|
import { objectCamelToSnake } from 'src/utils/CaseConverter';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class PipelinesService {
|
||||||
|
constructor(private pipelineClient: PipelinesClientService) {}
|
||||||
|
|
||||||
|
async getPipelineStatus(data: IIdRequest) {
|
||||||
|
try {
|
||||||
|
const pipelineStatusResponse =
|
||||||
|
await this.pipelineClient.getPipelineStatus(data);
|
||||||
|
|
||||||
|
return objectCamelToSnake(pipelineStatusResponse);
|
||||||
|
} catch (err) {
|
||||||
|
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async runPipeline({ id, info }: IIdRequest) {
|
||||||
|
try {
|
||||||
|
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
|
||||||
|
id,
|
||||||
|
info,
|
||||||
|
});
|
||||||
|
|
||||||
|
return objectCamelToSnake(triggerPipelineResponse);
|
||||||
|
} catch (err) {
|
||||||
|
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||||
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
|
|
||||||
export class PipelineInputsDTO {
|
export class PipelineInputsDTO {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
@@ -61,12 +62,6 @@ export interface IIdRequest {
|
|||||||
info: Info;
|
info: Info;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface Info {
|
|
||||||
user_id: string;
|
|
||||||
customer_id: string;
|
|
||||||
customer: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IUpdatePipelineRequest {
|
export interface IUpdatePipelineRequest {
|
||||||
input: IdRequest;
|
input: IdRequest;
|
||||||
transformations: IdRequest[];
|
transformations: IdRequest[];
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import {
|
|||||||
HttpException,
|
HttpException,
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
UseGuards,
|
UseGuards,
|
||||||
|
Res,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import {
|
import {
|
||||||
ApiCreatedResponse,
|
ApiCreatedResponse,
|
||||||
@@ -34,6 +35,7 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
|
|||||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||||
|
|
||||||
|
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
|
||||||
import {
|
import {
|
||||||
ICompleteUploadCSVFile,
|
ICompleteUploadCSVFile,
|
||||||
ICreatePipelineCSVFile,
|
ICreatePipelineCSVFile,
|
||||||
@@ -62,7 +64,9 @@ export class PipelinesController {
|
|||||||
constructor(
|
constructor(
|
||||||
@Inject(DadosferaLogger)
|
@Inject(DadosferaLogger)
|
||||||
dadosferaLogger: DadosferaLogger,
|
dadosferaLogger: DadosferaLogger,
|
||||||
|
|
||||||
private pipelinesClientService: PipelinesService,
|
private pipelinesClientService: PipelinesService,
|
||||||
|
private oldPipelinesService: OldPipelineService,
|
||||||
) {
|
) {
|
||||||
this.logger = dadosferaLogger.logger;
|
this.logger = dadosferaLogger.logger;
|
||||||
}
|
}
|
||||||
@@ -198,7 +202,7 @@ export class PipelinesController {
|
|||||||
customer: body.info.customer,
|
customer: body.info.customer,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = await this.pipelinesClientService.getPipelineStatus(body);
|
const response = await this.oldPipelinesService.getPipelineStatus(body);
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
@@ -509,7 +513,7 @@ export class PipelinesController {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
const response = await this.pipelinesClientService.runPipeline({ id, info });
|
const response = await this.oldPipelinesService.runPipeline({ id, info });
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { PipelinesService } from './pipelines.service';
|
|||||||
|
|
||||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
|
|
||||||
|
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||||
import { ConnectorModule } from '../connector/connector.module';
|
import { ConnectorModule } from '../connector/connector.module';
|
||||||
import { InputsModule } from '../inputs/inputs.module';
|
import { InputsModule } from '../inputs/inputs.module';
|
||||||
import { TransformationsModule } from '../transformations/transformations.module';
|
import { TransformationsModule } from '../transformations/transformations.module';
|
||||||
@@ -20,6 +21,7 @@ const client = new PipelinesClientConfiguration();
|
|||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
ClientsModule.register([client.providerOptions]),
|
ClientsModule.register([client.providerOptions]),
|
||||||
|
OldPipelineModule,
|
||||||
ConnectorModule,
|
ConnectorModule,
|
||||||
InputsModule,
|
InputsModule,
|
||||||
TransformationsModule,
|
TransformationsModule,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
/* eslint-disable no-async-promise-executor */
|
/* eslint-disable no-async-promise-executor */
|
||||||
import {
|
import {
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
ConflictException,
|
|
||||||
HttpException,
|
HttpException,
|
||||||
HttpStatus,
|
HttpStatus,
|
||||||
Inject,
|
Inject,
|
||||||
@@ -18,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
|
|||||||
|
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||||
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
||||||
import { Metadata } from '@grpc/grpc-js';
|
import { Metadata } from '@grpc/grpc-js';
|
||||||
import { ConnectorClientService } from '../connector/client.service';
|
import { ConnectorClientService } from '../connector/client.service';
|
||||||
@@ -633,49 +632,4 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
return assets;
|
return assets;
|
||||||
}
|
}
|
||||||
|
|
||||||
async getPipelineStatus(data) {
|
|
||||||
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
|
||||||
|
|
||||||
const statusPipelineResponse = await lastValueFrom(
|
|
||||||
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
|
|
||||||
)
|
|
||||||
.then((res) => {
|
|
||||||
const statusArray =
|
|
||||||
res.status?.sort((a, b) => {
|
|
||||||
if (a.id < b.id) {
|
|
||||||
return 1;
|
|
||||||
} else {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}) || [];
|
|
||||||
return { status: statusArray };
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
this.logger.error(err.message);
|
|
||||||
throw new Error(err);
|
|
||||||
});
|
|
||||||
this.logger.info('Done');
|
|
||||||
|
|
||||||
return statusPipelineResponse;
|
|
||||||
}
|
|
||||||
|
|
||||||
async runPipeline({ id, info }: IIdRequest) {
|
|
||||||
this.logger.info('PipelinesClientService - RunPipeline');
|
|
||||||
const statusPipelineResponse = await lastValueFrom(
|
|
||||||
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
|
|
||||||
).catch((err) => {
|
|
||||||
this.logger.error(err.message);
|
|
||||||
throw new Error(err);
|
|
||||||
});
|
|
||||||
|
|
||||||
if (statusPipelineResponse.status == false) {
|
|
||||||
throw new ConflictException(
|
|
||||||
'This pipeline is not ready yet to execute, Try again later!',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.logger.info('Done');
|
|
||||||
return statusPipelineResponse;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -832,24 +832,6 @@ export class PlatformApiController {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
|
|
||||||
@ApiOperation({ summary: 'Get pipeline run jobs' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
|
||||||
async getPipelineRunJobs(
|
|
||||||
@Param('pipelineId') pipelineId: string,
|
|
||||||
@Param('runId') runId: string,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
) {
|
|
||||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
|
||||||
const normalizedRunId = this.normalizePipelineId(runId);
|
|
||||||
|
|
||||||
return this.platformApiService.proxy(
|
|
||||||
'GET',
|
|
||||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`,
|
|
||||||
user,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== JOBS - COLUMN EDITING ROUTES ====================
|
// ==================== JOBS - COLUMN EDITING ROUTES ====================
|
||||||
|
|
||||||
@Put('jobs/:jobId/input')
|
@Put('jobs/:jobId/input')
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { ReleaseNoteController } from './release_note.controller';
|
import { ReleaseNoteController } from './release_note.controller';
|
||||||
import { ReleaseNoteService } from './release_note.service';
|
import { ReleaseNoteService } from './release_note.service';
|
||||||
|
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
describe('ReleaseNoteController', () => {
|
describe('ReleaseNoteController', () => {
|
||||||
let controller: ReleaseNoteController;
|
let controller: ReleaseNoteController;
|
||||||
@@ -8,7 +9,7 @@ describe('ReleaseNoteController', () => {
|
|||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
controllers: [ReleaseNoteController],
|
controllers: [ReleaseNoteController],
|
||||||
providers: [ReleaseNoteService],
|
providers: [ReleaseNoteService, DadosferaLogger],
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { ReleaseNoteService } from './release_note.service';
|
import { ReleaseNoteService } from './release_note.service';
|
||||||
|
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
describe('ReleaseNoteService', () => {
|
describe('ReleaseNoteService', () => {
|
||||||
let service: ReleaseNoteService;
|
let service: ReleaseNoteService;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
providers: [ReleaseNoteService],
|
providers: [ReleaseNoteService, DadosferaLogger],
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
||||||
|
|||||||
+2
-5
@@ -1,8 +1,5 @@
|
|||||||
export interface Info {
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
user_id: string;
|
|
||||||
customer_id: string;
|
|
||||||
customer: string;
|
|
||||||
}
|
|
||||||
export interface ICreateTransformationsRequest {
|
export interface ICreateTransformationsRequest {
|
||||||
transformations: Transformation[];
|
transformations: Transformation[];
|
||||||
info: Info;
|
info: Info;
|
||||||
|
|||||||
Reference in New Issue
Block a user