Compare commits

..
21 changed files with 399 additions and 141 deletions
-6
View File
@@ -71,11 +71,6 @@ jobs:
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff plugin
run: |
helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
helm diff version
- name: Debug Helm env
run: |
helm env
@@ -107,5 +102,4 @@ jobs:
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
HELM_PLUGINS: /home/runner/.local/share/helm/plugins
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
+90 -45
View File
@@ -3343,7 +3343,14 @@
],
"responses": {
"200": {
"description": ""
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
@@ -3867,6 +3874,88 @@
]
}
},
"/pipelines/start/{id}": {
"post": {
"operationId": "PipelinesController_activate",
"summary": "",
"deprecated": true,
"description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Pipelines"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/pipelines/{id}/status": {
"get": {
"operationId": "PipelinesController_getPipelineStatus",
"summary": "",
"deprecated": true,
"description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Pipelines"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/transformations": {
"post": {
"operationId": "TransformationsController_create",
@@ -4570,50 +4659,6 @@
]
}
},
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": {
"get": {
"operationId": "PlatformApiController_getPipelineRunJobs",
"summary": "Get pipeline run jobs",
"parameters": [
{
"name": "pipelineId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "runId",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"Platform API"
],
"security": [
{
"access-token": []
}
]
}
},
"/platform/jobs/{jobId}/input": {
"put": {
"operationId": "PlatformApiController_updateJobInput",
+14 -4
View File
@@ -16,7 +16,8 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.40.0-beta.10",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.9",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1734,10 +1735,19 @@
"winston-log2gelf": "^2.4.0"
}
},
"node_modules/@dadosfera/protospack": {
"version": "2.5.3",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
"license": "ISC",
"dependencies": {
"rxjs": "^7.5.5"
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.40.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
"integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
"version": "3.40.0-beta.9",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.9.tgz",
"integrity": "sha512-8jbCpzxQnDax41yhw8yQJLB1VVp4PY8grj2H5oBgV+BCUf2y5lp0EPzLwVZsnOsfsXNAXHyNmbZbAjmJC1ipaw==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
+2 -1
View File
@@ -34,7 +34,8 @@
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.40.0-beta.10",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.9",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+2
View File
@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
import { NetworkConfigModule } from './modules/network-config/network-config.module';
import { InputsModule } from './modules/inputs/inputs.module';
import { OauthModule } from './modules/oauth/oauth.module';
import { PipelinesModule } from './modules/pipelines/pipelines.module';
import { TransformationsModule } from './modules/transformations/transformations.module';
import { HealthModule } from './modules/health/health.module';
import { CatalogModule } from './modules/catalog/catalog.module';
@@ -59,6 +60,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
PermissionsModule,
TermsOfUseModule,
ConnectionTestModule,
PipelinesModule,
TransformationsModule,
UsersModule,
RolesModule,
+3
View File
@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
+1 -5
View File
@@ -1,8 +1,4 @@
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
interface Values {
jdbc_user: string;
+78
View File
@@ -0,0 +1,78 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.pipelineService =
this.grpcClient.getService<PipelinesServiceInterface>(
PipelineServicesNames.PipelineService,
);
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.getPipelineStatus(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.triggerPipeline({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
}
+36
View File
@@ -0,0 +1,36 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
info: Info;
}
export interface IdRequest {
id: string;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
id: string;
info: Info;
}
export interface IGetPipelineLogsRequest {
id: string;
details: string;
}
+33
View File
@@ -0,0 +1,33 @@
import {
ClientsProviderAsyncOptions,
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
import { credentials } from '@grpc/grpc-js';
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
process.env.PIFACTORY_URL.includes('0.0.0.0');
export class PipelinesClientConfiguration {
public name = 'PipelinesClientConfiguration';
private config: GrpcOptions = {
transport: Transport.GRPC,
options: {
url: process.env.PIFACTORY_URL,
package: PipelinePackages,
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: PipelineProtoFilePath,
loader: {
keepCase: true,
enums: String,
defaults: false,
},
},
};
providerOptions: ClientsProviderAsyncOptions = {
name: this.name,
...this.config,
};
}
@@ -0,0 +1,72 @@
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private pipelineService: PipelinesService,
) {
this.logger = dadosferaLogger.logger;
}
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
})
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
this.logger.info(
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.runPipeline({ id, info });
return response;
}
@Get(':id/status')
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.getPipelineStatus(body);
return response;
}
}
+19
View File
@@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesController } from './pipelines.controller';
import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesClientService } from './client.service';
const client = new PipelinesClientConfiguration();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [PipelinesController],
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
exports: [PipelinesService],
})
export class PipelinesModule {}
@@ -0,0 +1,33 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { PipelinesClientService } from './client.service';
import { IIdRequest } from './interfaces';
import { objectCamelToSnake } from 'src/utils/CaseConverter';
@Injectable()
export class PipelinesService {
constructor(private pipelineClient: PipelinesClientService) {}
async getPipelineStatus(data: IIdRequest) {
try {
const pipelineStatusResponse =
await this.pipelineClient.getPipelineStatus(data);
return objectCamelToSnake(pipelineStatusResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async runPipeline({ id, info }: IIdRequest) {
try {
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
id,
info,
});
return objectCamelToSnake(triggerPipelineResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
}
+1 -6
View File
@@ -1,4 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export class PipelineInputsDTO {
@ApiProperty()
@@ -61,12 +62,6 @@ export interface IIdRequest {
info: Info;
}
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
@@ -15,6 +15,7 @@ import {
HttpException,
BadRequestException,
UseGuards,
Res,
} from '@nestjs/common';
import {
ApiCreatedResponse,
@@ -34,6 +35,7 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
ICompleteUploadCSVFile,
ICreatePipelineCSVFile,
@@ -62,7 +64,9 @@ export class PipelinesController {
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private pipelinesClientService: PipelinesService,
private oldPipelinesService: OldPipelineService,
) {
this.logger = dadosferaLogger.logger;
}
@@ -198,7 +202,7 @@ export class PipelinesController {
customer: body.info.customer,
});
const response = await this.pipelinesClientService.getPipelineStatus(body);
const response = await this.oldPipelinesService.getPipelineStatus(body);
return response;
}
@@ -509,7 +513,7 @@ export class PipelinesController {
},
);
const response = await this.pipelinesClientService.runPipeline({ id, info });
const response = await this.oldPipelinesService.runPipeline({ id, info });
return response;
}
@@ -7,6 +7,7 @@ import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { ConnectorModule } from '../connector/connector.module';
import { InputsModule } from '../inputs/inputs.module';
import { TransformationsModule } from '../transformations/transformations.module';
@@ -20,6 +21,7 @@ const client = new PipelinesClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
ConnectorModule,
InputsModule,
TransformationsModule,
+1 -47
View File
@@ -1,7 +1,6 @@
/* eslint-disable no-async-promise-executor */
import {
BadRequestException,
ConflictException,
HttpException,
HttpStatus,
Inject,
@@ -18,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service';
@@ -633,49 +632,4 @@ export class PipelinesService implements OnModuleInit {
return assets;
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
}
@@ -832,24 +832,6 @@ export class PlatformApiController {
);
}
@Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
@ApiOperation({ summary: 'Get pipeline run jobs' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineRunJobs(
@Param('pipelineId') pipelineId: string,
@Param('runId') runId: string,
@User() user: RequestUser,
) {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const normalizedRunId = this.normalizePipelineId(runId);
return this.platformApiService.proxy(
'GET',
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`,
user,
);
}
// ==================== JOBS - COLUMN EDITING ROUTES ====================
@Put('jobs/:jobId/input')
@@ -1,6 +1,7 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteController } from './release_note.controller';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteController', () => {
let controller: ReleaseNoteController;
@@ -8,7 +9,7 @@ describe('ReleaseNoteController', () => {
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [ReleaseNoteController],
providers: [ReleaseNoteService],
providers: [ReleaseNoteService, DadosferaLogger],
}).compile();
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
@@ -1,12 +1,13 @@
import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteService', () => {
let service: ReleaseNoteService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [ReleaseNoteService],
providers: [ReleaseNoteService, DadosferaLogger],
}).compile();
service = module.get<ReleaseNoteService>(ReleaseNoteService);
+2 -5
View File
@@ -1,8 +1,5 @@
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreateTransformationsRequest {
transformations: Transformation[];
info: Info;