Compare commits

..
30 changed files with 442 additions and 794 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
FROM node:20-alpine AS base_image FROM node:20-alpine AS base_image
RUN npm install -g npm@10.8.2 RUN npm install -g npm@latest
FROM base_image AS build_base FROM base_image AS build_base
WORKDIR /app WORKDIR /app
+87 -428
View File
File diff suppressed because it is too large Load Diff
+15 -4
View File
@@ -16,7 +16,8 @@
"@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0", "@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.14", "@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.8",
"@grpc/grpc-js": "^1.9.3", "@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9", "@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0", "@nestjs/cli": "^9.5.0",
@@ -1734,10 +1735,20 @@
"winston-log2gelf": "^2.4.0" "winston-log2gelf": "^2.4.0"
} }
}, },
"node_modules/@dadosfera/protospack": {
"version": "2.5.3",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
"license": "ISC",
"dependencies": {
"rxjs": "^7.5.5"
}
},
"node_modules/@dadosfera/protospack-v2": { "node_modules/@dadosfera/protospack-v2": {
"version": "3.40.0-beta.14", "version": "3.40.0-beta.8",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz",
"integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==", "integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==",
"license": "ISC",
"dependencies": { "dependencies": {
"@grpc/grpc-js": "^1.9.3", "@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5" "rxjs": "^7.5.5"
+2 -1
View File
@@ -34,7 +34,8 @@
"@aws-sdk/lib-dynamodb": "^3.414.0", "@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0", "@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.40.0-beta.14", "@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.8",
"@grpc/grpc-js": "^1.9.3", "@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9", "@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0", "@nestjs/cli": "^9.5.0",
+2
View File
@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
import { NetworkConfigModule } from './modules/network-config/network-config.module'; import { NetworkConfigModule } from './modules/network-config/network-config.module';
import { InputsModule } from './modules/inputs/inputs.module'; import { InputsModule } from './modules/inputs/inputs.module';
import { OauthModule } from './modules/oauth/oauth.module'; import { OauthModule } from './modules/oauth/oauth.module';
import { PipelinesModule } from './modules/pipelines/pipelines.module';
import { TransformationsModule } from './modules/transformations/transformations.module'; import { TransformationsModule } from './modules/transformations/transformations.module';
import { HealthModule } from './modules/health/health.module'; import { HealthModule } from './modules/health/health.module';
import { CatalogModule } from './modules/catalog/catalog.module'; import { CatalogModule } from './modules/catalog/catalog.module';
@@ -59,6 +60,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
PermissionsModule, PermissionsModule,
TermsOfUseModule, TermsOfUseModule,
ConnectionTestModule, ConnectionTestModule,
PipelinesModule,
TransformationsModule, TransformationsModule,
UsersModule, UsersModule,
RolesModule, RolesModule,
-12
View File
@@ -357,16 +357,6 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Crear y editar atributos en el catálogo', 'es-es': 'Crear y editar atributos en el catálogo',
}, },
}, },
CERTIFY: {
seqid: 53,
claim: 'catalog:certify',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Alterar o status de certificação dos Ativos',
'en-us': "Change Assets' certification status",
'es-es': 'Cambiar el estado de certificación de los Activos',
},
},
DELETE: { DELETE: {
seqid: 1, seqid: 1,
claim: 'catalog:delete', claim: 'catalog:delete',
@@ -722,8 +712,6 @@ export const DADOSFERA_MODULES_KEYS = {
PII: 'pii', PII: 'pii',
EMBED: 'embedded-analytics', EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned', EMBED_ASSIGNED: 'embed-assigned',
CATALOG: 'catalog',
COLLECT: 'collect',
} }
export const DADOSFERA_MODULES: Array<DadosferaModule> = [ export const DADOSFERA_MODULES: Array<DadosferaModule> = [
-102
View File
@@ -17,7 +17,6 @@ import {
HttpStatus, HttpStatus,
Res, Res,
} from '@nestjs/common'; } from '@nestjs/common';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import { import {
ApiCreatedResponse, ApiCreatedResponse,
ApiHeaders, ApiHeaders,
@@ -47,7 +46,6 @@ import {
IMakeAComment, IMakeAComment,
IOneDataAsset, IOneDataAsset,
IPreviewResponse, IPreviewResponse,
IUpdateCertificationStatusRequest,
IUpdateDataRequest, IUpdateDataRequest,
TriggerCatalogReq, TriggerCatalogReq,
TriggerCatalogRes, TriggerCatalogRes,
@@ -85,9 +83,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async searchCatalog( async searchCatalog(
@User() user: RequestUser, @User() user: RequestUser,
@Query() query: ICatalogAllRequest, @Query() query: ICatalogAllRequest,
@@ -127,9 +122,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async dowloadAsserts( async dowloadAsserts(
@User() user: RequestUser, @User() user: RequestUser,
@Query() query: ICatalogAllRequest, @Query() query: ICatalogAllRequest,
@@ -173,9 +165,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint() @ApiInternalOnlyEndpoint()
@Get('data-asset') @Get('data-asset')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) { async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user; const { username, user_id, customer_id, customer_name, permissions } = user;
const { pipeline, object } = query; const { pipeline, object } = query;
@@ -234,9 +223,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async findAllTags(@Body() body) { async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, { this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id, user: body.info.user_id,
@@ -283,31 +269,11 @@ export class CatalogController {
} }
@Get('custom-properties')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async getCustomPropertyDefinitions(@User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
return this.catalogService.getCustomPropertyDefinitions(metadata);
}
@Get('data-asset/:id') @Get('data-asset/:id')
@RequireSomePermission( @RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAsset( async getDataAsset(
@User() user: RequestUser, @User() user: RequestUser,
@Param('id') id: string, @Param('id') id: string,
@@ -419,9 +385,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetColumnsMetadata( async getDataAssetColumnsMetadata(
@User() user: RequestUser, @User() user: RequestUser,
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@@ -453,9 +416,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetPreview( async getDataAssetPreview(
@User() user: RequestUser, @User() user: RequestUser,
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@@ -487,9 +447,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.GET, PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getDataAssetDocs( async getDataAssetDocs(
@User() user: RequestUser, @User() user: RequestUser,
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@@ -521,9 +478,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAsset( async updateDataAsset(
@User() user: RequestUser, @User() user: RequestUser,
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@@ -539,8 +493,6 @@ export class CatalogController {
language, language,
}); });
delete (body as any).certification_status;
const result = await this.catalogService.updateOneDataAsset({ const result = await this.catalogService.updateOneDataAsset({
body, body,
data_asset_id, data_asset_id,
@@ -554,44 +506,11 @@ export class CatalogController {
return result; return result;
} }
@Put('data-asset/:id/certification-status')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async updateDataAssetCertificationStatus(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') data_asset_id: string,
@Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
): Promise<IUpdateCertificationStatusRequest> {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
return this.catalogService.updateCertificationStatus({
body,
data_asset_id,
metadata,
});
}
@Post('data-asset/:id/docs') @Post('data-asset/:id/docs')
@RequireSomePermission( @RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetDocs( async manageDataAssetDocs(
@User() user: RequestUser, @User() user: RequestUser,
@Headers() headers, @Headers() headers,
@@ -629,9 +548,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint() @ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions') @Put('data-asset/:id/manage-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async manageDataAssetPermissions( async manageDataAssetPermissions(
@Param('id') id: string, @Param('id') id: string,
@User() user: RequestUser, @User() user: RequestUser,
@@ -654,9 +570,6 @@ export class CatalogController {
@ApiInternalOnlyEndpoint() @ApiInternalOnlyEndpoint()
@Put('data-asset/:id/revoke-permissions') @Put('data-asset/:id/revoke-permissions')
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async revokeDataAssetPermissions( async revokeDataAssetPermissions(
@Param('id') id: string, @Param('id') id: string,
@User() user: RequestUser, @User() user: RequestUser,
@@ -682,9 +595,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE, PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async createDataAsset( async createDataAsset(
@User() user: RequestUser, @User() user: RequestUser,
@Body() body: ICreateDataAsset, @Body() body: ICreateDataAsset,
@@ -709,9 +619,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async commentOnDataAsset( async commentOnDataAsset(
@Param('id') id: string, @Param('id') id: string,
@User() user: RequestUser, @User() user: RequestUser,
@@ -738,9 +645,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE, PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) { async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user; const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({ const metadata = PackTheMetadata({
@@ -762,9 +666,6 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE, PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async deleteComment( async deleteComment(
@Param('id') id: string, @Param('id') id: string,
@User() user: RequestUser, @User() user: RequestUser,
@@ -918,9 +819,6 @@ export class CatalogController {
@Get('nimbus-dashboards') @Get('nimbus-dashboards')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER) @RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
@RequireModule(
DADOSFERA_MODULES_KEYS.CATALOG
)
async getNimbusDashboards( async getNimbusDashboards(
@User() user: RequestUser, @User() user: RequestUser,
@Body() body: GetNimbusDashboardsRequest, @Body() body: GetNimbusDashboardsRequest,
+3
View File
@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller'; import { CatalogController } from './catalog.controller';
import { CatalogClientConfiguration } from './catalog-client'; import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices'; import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module'; import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module'; import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module'; import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module'; import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service'; import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration(); const client = new CatalogClientConfiguration();
@Module({ @Module({
imports: [ imports: [
ClientsModule.register([client.providerOptions]), ClientsModule.register([client.providerOptions]),
OldPipelineModule,
UsersModule, UsersModule,
RolesModule, RolesModule,
CustomersModule, CustomersModule,
-27
View File
@@ -29,7 +29,6 @@ import {
AssetReporter, AssetReporter,
BatchRemoveRlsRulesRequest, BatchRemoveRlsRulesRequest,
CreateDataDocsDTO, CreateDataDocsDTO,
IUpdateCertificationStatusRequest,
IUpdateDataRequest, IUpdateDataRequest,
TriggerCatalogReq, TriggerCatalogReq,
} from './dtos'; } from './dtos';
@@ -120,10 +119,6 @@ class CatalogService implements OnModuleInit {
} }
} }
async getCustomPropertyDefinitions(metadata: Metadata) {
return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata));
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions'); this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined; if (!data.embed) data.embed = undefined;
@@ -389,28 +384,6 @@ class CatalogService implements OnModuleInit {
return { data_asset: asset[0] }; return { data_asset: asset[0] };
} }
async updateCertificationStatus(data: {
data_asset_id: string;
body: IUpdateCertificationStatusRequest;
metadata: Metadata;
}) {
const { body, data_asset_id, metadata } = data;
await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{
id: data_asset_id,
changes: JSON.stringify({
certification_status: body.certification_status,
}),
},
metadata,
),
);
return { certification_status: body.certification_status };
}
async updateOneDataAsset(data: { async updateOneDataAsset(data: {
data_asset_id: string; data_asset_id: string;
customer_id: string; customer_id: string;
-42
View File
@@ -1,10 +1,4 @@
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger'; import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
import {
IsEnum,
IsNotEmpty,
IsOptional,
IsString,
} from 'class-validator';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export enum DataAssetShareType { export enum DataAssetShareType {
@@ -12,12 +6,6 @@ export enum DataAssetShareType {
public = 'public', public = 'public',
private = 'private', private = 'private',
} }
export enum CertificationStatus {
draft = 'draft',
in_review = 'in_review',
approved = 'approved',
deprecated = 'deprecated',
}
export enum OrderEnum { export enum OrderEnum {
asc = 'asc', asc = 'asc',
desc = 'desc', desc = 'desc',
@@ -203,27 +191,6 @@ export class IData {
day_opening: number; day_opening: number;
} }
export enum CustomPropertyType {
TEXT = 'text',
NUMBER = 'number',
DATE = 'date',
BOOLEAN = 'boolean',
}
export class CustomPropertyDto {
@ApiProperty()
key: string;
@ApiProperty()
value: string;
@ApiProperty({ enum: CustomPropertyType })
type: CustomPropertyType;
@ApiPropertyOptional()
color?: string;
@ApiPropertyOptional()
emoji?: string;
}
export class IUpdateDataRequest { export class IUpdateDataRequest {
@ApiProperty() @ApiProperty()
name: string; name: string;
@@ -237,16 +204,7 @@ export class IUpdateDataRequest {
share_type?: DataAssetShareType; share_type?: DataAssetShareType;
@ApiPropertyOptional() @ApiPropertyOptional()
docs?: string; docs?: string;
@ApiPropertyOptional({ type: [CustomPropertyDto] })
custom_properties?: CustomPropertyDto[];
} }
export class IUpdateCertificationStatusRequest {
@ApiProperty({ enum: CertificationStatus })
@IsEnum(CertificationStatus)
certification_status: CertificationStatus;
}
export class ICreateDataAsset implements CreateDataAssetRequest { export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty() @ApiProperty()
display_name: string; display_name: string;
@@ -24,19 +24,15 @@ import {
GetTableMetadataReq, GetTableMetadataReq,
} from './dto/connection-test'; } from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator'; import { Authenticated } from 'src/decorators/authentication.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum';
@ApiInternalOnlyController() @ApiInternalOnlyController()
@ApiTags('Connection Test') @ApiTags('Connection Test')
@Controller('connection-test') @Controller('connection-test')
@UseFilters(new GrpcToHttpExceptionFilter()) @UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated() @Authenticated()
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionTestController { export class ConnectionTestController {
logger: any; logger: any;
constructor( constructor(
@@ -16,9 +16,8 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { import {
Authenticated, Authenticated,
RequireAllPermissions, RequireAllPermissions,
RequireModule,
} from 'src/decorators/authentication.decorator'; } from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/decorators/user.decorator'; import { RequestUser, User } from 'src/decorators/user.decorator';
import { ValidationPipe } from '../../pipes/object-validation.pipe'; import { ValidationPipe } from '../../pipes/object-validation.pipe';
import { import {
@@ -40,9 +39,6 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
@ApiTags('connections') @ApiTags('connections')
@Authenticated() @Authenticated()
@Controller('connections') @Controller('connections')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class ConnectionController { export class ConnectionController {
logger: any; logger: any;
constructor( constructor(
+1 -26
View File
@@ -25,10 +25,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { import {
Authenticated, Authenticated,
RequireAllPermissions, RequireAllPermissions,
RequireModule,
RequireSomePermission, RequireSomePermission,
} from 'src/decorators/authentication.decorator'; } from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Language } from 'src/decorators/language.decorator'; import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum'; import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@@ -100,9 +99,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getAllConnectors( async getAllConnectors(
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@Query() queries: GetAllDto, @Query() queries: GetAllDto,
@@ -135,9 +131,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorsTags() { async getConnectorsTags() {
return await this.connectorClientService.getConnectorsTags(); return await this.connectorClientService.getConnectorsTags();
} }
@@ -150,9 +143,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnector( async getConnector(
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@Param('plugin') plugin: string, @Param('plugin') plugin: string,
@@ -181,9 +171,6 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE, PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
) )
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async getConnectorDetails( async getConnectorDetails(
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@Param('plugin') plugin: string, @Param('plugin') plugin: string,
@@ -206,9 +193,6 @@ export class ConnectorController {
@Put('/:plugin') @Put('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE) @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@ApiConsumes('multipart/form-data') @ApiConsumes('multipart/form-data')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async updateConnector( async updateConnector(
@Param('plugin') plugin: string, @Param('plugin') plugin: string,
@Body() body: UpdateDto, @Body() body: UpdateDto,
@@ -230,9 +214,6 @@ export class ConnectorController {
@Put('/:plugin/add-tag') @Put('/:plugin/add-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE) @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async addTagOnConnector( async addTagOnConnector(
@Param('plugin') plugin: string, @Param('plugin') plugin: string,
@Body() body: AddTagDto, @Body() body: AddTagDto,
@@ -260,9 +241,6 @@ export class ConnectorController {
@Put('/:plugin/remove-tag') @Put('/:plugin/remove-tag')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE) @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async removeTagOnConnector( async removeTagOnConnector(
@Param('plugin') plugin: string, @Param('plugin') plugin: string,
@Body() body: RemoveTagDto, @Body() body: RemoveTagDto,
@@ -291,9 +269,6 @@ export class ConnectorController {
@Delete('/:plugin') @Delete('/:plugin')
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE) @RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
async deleteConnector( async deleteConnector(
@Param('plugin') plugin: string, @Param('plugin') plugin: string,
@Query('version') version: string, @Query('version') version: string,
+1 -5
View File
@@ -1,8 +1,4 @@
export interface Info { import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
user_id: string;
customer_id: string;
customer: string;
}
interface Values { interface Values {
jdbc_user: string; jdbc_user: string;
+4 -5
View File
@@ -23,7 +23,6 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages'; } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { CreateInputReq } from './dtos/input.model'; import { CreateInputReq } from './dtos/input.model';
import { Metadata } from '@grpc/grpc-js';
@Injectable() @Injectable()
@@ -74,10 +73,10 @@ export class InputsService {
objectCamelToSnake(createInputResponse); objectCamelToSnake(createInputResponse);
return createInputResponse; return createInputResponse;
}, },
update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => { update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO)); this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
const updateInputResponse = await lastValueFrom( const updateInputResponse = await lastValueFrom(
this.inputWriteService.InputUpdate(updateInputDTO, metadata), this.inputWriteService.InputUpdate(updateInputDTO),
); );
return updateInputResponse; return updateInputResponse;
@@ -207,7 +206,7 @@ export class InputsService {
return findOneInputResponse; return findOneInputResponse;
} }
async update(id: string, data, info: Info, metadata?: Metadata) { async update(id: string, data, info: Info) {
// this.validateCron({ ...data, info }); // this.validateCron({ ...data, info });
try { try {
const { const {
@@ -218,7 +217,7 @@ export class InputsService {
id, id,
...data, ...data,
info, info,
}, metadata); });
const updateInputResponse = this.adjustInputPayload( const updateInputResponse = this.adjustInputPayload(
input, input,
+78
View File
@@ -0,0 +1,78 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.pipelineService =
this.grpcClient.getService<PipelinesServiceInterface>(
PipelineServicesNames.PipelineService,
);
}
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.getPipelineStatus(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineService.triggerPipeline({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
}
+36
View File
@@ -0,0 +1,36 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
info: Info;
}
export interface IdRequest {
id: string;
}
export interface IIdRequest {
id: string;
info: Info;
}
export interface IUpdatePipelineRequest {
input: IdRequest;
transformations: IdRequest[];
output: IdRequest;
tags: string[];
name: string;
description: string;
id: string;
info: Info;
}
export interface IGetPipelineLogsRequest {
id: string;
details: string;
}
+33
View File
@@ -0,0 +1,33 @@
import {
ClientsProviderAsyncOptions,
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
import { credentials } from '@grpc/grpc-js';
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
process.env.PIFACTORY_URL.includes('0.0.0.0');
export class PipelinesClientConfiguration {
public name = 'PipelinesClientConfiguration';
private config: GrpcOptions = {
transport: Transport.GRPC,
options: {
url: process.env.PIFACTORY_URL,
package: PipelinePackages,
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: PipelineProtoFilePath,
loader: {
keepCase: true,
enums: String,
defaults: false,
},
},
};
providerOptions: ClientsProviderAsyncOptions = {
name: this.name,
...this.config,
};
}
@@ -0,0 +1,72 @@
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private pipelineService: PipelinesService,
) {
this.logger = dadosferaLogger.logger;
}
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
})
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
this.logger.info(
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.runPipeline({ id, info });
return response;
}
@Get(':id/status')
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.pipelineService.getPipelineStatus(body);
return response;
}
}
+19
View File
@@ -0,0 +1,19 @@
import { Module } from '@nestjs/common';
import { ClientsModule } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesController } from './pipelines.controller';
import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesClientService } from './client.service';
const client = new PipelinesClientConfiguration();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [PipelinesController],
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
exports: [PipelinesService],
})
export class PipelinesModule {}
@@ -0,0 +1,33 @@
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { PipelinesClientService } from './client.service';
import { IIdRequest } from './interfaces';
import { objectCamelToSnake } from 'src/utils/CaseConverter';
@Injectable()
export class PipelinesService {
constructor(private pipelineClient: PipelinesClientService) {}
async getPipelineStatus(data: IIdRequest) {
try {
const pipelineStatusResponse =
await this.pipelineClient.getPipelineStatus(data);
return objectCamelToSnake(pipelineStatusResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
async runPipeline({ id, info }: IIdRequest) {
try {
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
id,
info,
});
return objectCamelToSnake(triggerPipelineResponse);
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
}
+1 -6
View File
@@ -1,4 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export class PipelineInputsDTO { export class PipelineInputsDTO {
@ApiProperty() @ApiProperty()
@@ -61,12 +62,6 @@ export interface IIdRequest {
info: Info; info: Info;
} }
export interface Info {
user_id: string;
customer_id: string;
customer: string;
}
export interface IUpdatePipelineRequest { export interface IUpdatePipelineRequest {
input: IdRequest; input: IdRequest;
transformations: IdRequest[]; transformations: IdRequest[];
+14 -23
View File
@@ -25,16 +25,16 @@ import {
} from '@nestjs/swagger'; } from '@nestjs/swagger';
import { import {
RequireAllPermissions, RequireAllPermissions,
RequireModule,
RequireSomePermission, RequireSomePermission,
} from 'src/decorators/authentication.decorator'; } from 'src/decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service'; import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator'; import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata'; import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import { import {
ICompleteUploadCSVFile, ICompleteUploadCSVFile,
ICreatePipelineCSVFile, ICreatePipelineCSVFile,
@@ -58,15 +58,14 @@ type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter()) @UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2') @Controller('pipelinesV2')
@RequireModule(
DADOSFERA_MODULES_KEYS.COLLECT
)
export class PipelinesController { export class PipelinesController {
logger: DadosferaLogger; logger: DadosferaLogger;
constructor( constructor(
@Inject(DadosferaLogger) @Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger, dadosferaLogger: DadosferaLogger,
private pipelinesClientService: PipelinesService, private pipelinesClientService: PipelinesService,
private oldPipelinesService: OldPipelineService,
) { ) {
this.logger = dadosferaLogger.logger; this.logger = dadosferaLogger.logger;
} }
@@ -202,7 +201,7 @@ export class PipelinesController {
customer: body.info.customer, customer: body.info.customer,
}); });
const response = await this.pipelinesClientService.getPipelineStatus(body); const response = await this.oldPipelinesService.getPipelineStatus(body);
return response; return response;
} }
@@ -321,6 +320,7 @@ export class PipelinesController {
) { ) {
this.logger.info('PipelinesController - update', { user }); this.logger.info('PipelinesController - update', { user });
const { customer_id, customer_name, user_id, username } = user;
const info: Info = { const info: Info = {
user_id: user.user_id, user_id: user.user_id,
customer: user.customer_name, customer: user.customer_name,
@@ -328,7 +328,13 @@ export class PipelinesController {
pipeline_id: pipelineId pipeline_id: pipelineId
}; };
const metadata = PackTheMetadata(user); const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
const response = await this.pipelinesClientService.updatePipelineInput( const response = await this.pipelinesClientService.updatePipelineInput(
pipelineId, pipelineId,
@@ -362,21 +368,6 @@ export class PipelinesController {
return response; return response;
} }
@Patch('/:id/upgrade')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@HttpCode(HttpStatus.NO_CONTENT)
async upgradeConnector(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
) {
this.logger.info('PipelinesController - upgrade connector');
const metadata = PackTheMetadata(user);
await this.pipelinesClientService.upgrade(id, metadata);
}
@Delete(':id') @Delete(':id')
@ApiNoContentResponse() @ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT) @HttpCode(HttpStatus.NO_CONTENT)
@@ -506,7 +497,7 @@ export class PipelinesController {
}, },
); );
const response = await this.pipelinesClientService.runPipeline({ id, info }); const response = await this.oldPipelinesService.runPipeline({ id, info });
return response; return response;
} }
@@ -7,6 +7,7 @@ import { PipelinesService } from './pipelines.service';
import { PipelinesClientConfiguration } from './pipelines-client'; import { PipelinesClientConfiguration } from './pipelines-client';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { ConnectorModule } from '../connector/connector.module'; import { ConnectorModule } from '../connector/connector.module';
import { InputsModule } from '../inputs/inputs.module'; import { InputsModule } from '../inputs/inputs.module';
import { TransformationsModule } from '../transformations/transformations.module'; import { TransformationsModule } from '../transformations/transformations.module';
@@ -20,6 +21,7 @@ const client = new PipelinesClientConfiguration();
@Module({ @Module({
imports: [ imports: [
ClientsModule.register([client.providerOptions]), ClientsModule.register([client.providerOptions]),
OldPipelineModule,
ConnectorModule, ConnectorModule,
InputsModule, InputsModule,
TransformationsModule, TransformationsModule,
+29 -87
View File
@@ -1,7 +1,6 @@
/* eslint-disable no-async-promise-executor */ /* eslint-disable no-async-promise-executor */
import { import {
BadRequestException, BadRequestException,
ConflictException,
HttpException, HttpException,
HttpStatus, HttpStatus,
Inject, Inject,
@@ -18,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client'; import { PipelinesClientConfiguration } from './pipelines-client';
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages'; import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { Metadata } from '@grpc/grpc-js'; import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service'; import { ConnectorClientService } from '../connector/client.service';
@@ -177,15 +176,6 @@ export class PipelinesService implements OnModuleInit {
return updatePipelineResponse; return updatePipelineResponse;
} }
async upgrade(id: string, metadata: Metadata) {
await lastValueFrom(
this.pipelineWriteService.Upgrade(
{ id },
metadata,
),
);
}
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) { async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
const { id, metadata, user } = data; const { id, metadata, user } = data;
const info = { const info = {
@@ -383,8 +373,7 @@ export class PipelinesService implements OnModuleInit {
const updateInputResponse = await this.inputsService.update( const updateInputResponse = await this.inputsService.update(
inputId, inputId,
updateInputDTO, updateInputDTO,
info, info
metadata
); );
const inputRollback = () => { const inputRollback = () => {
@@ -405,36 +394,34 @@ export class PipelinesService implements OnModuleInit {
const nimbusUpdates = updateInputResponse?.tablesUpdate || []; const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
if (user.customer_modules.includes('catalog')) { nimbusUpdates.forEach(update => {
nimbusUpdates.forEach(update => { const nimbusRollback = () => {
const nimbusRollback = () => { return this.nimbusService.renameTable(
return this.nimbusService.renameTable( info.customer,
info.customer, update.database,
update.database, {
{ table_name: update.table_name,
table_name: update.table_name, table_schema: update.table_schema
table_schema: update.table_schema },
}, {
{ table_name: update.old_table_name,
table_name: update.old_table_name, table_schema: update.old_table_schema
table_schema: update.old_table_schema }
} );
);
}
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
} }
rollback.push(nimbusRollback);
});
try {
await this.updateNimbus(info.customer, nimbusUpdates);
} catch (error) {
this.logger.error(error);
if (error instanceof AxiosError) {
this.logger.error(JSON.stringify(error.response.data));
}
await this.executeRenameRollback(rollback);
throw new Error("Error Nimbus updating tables");
} }
try { try {
@@ -636,49 +623,4 @@ export class PipelinesService implements OnModuleInit {
return assets; return assets;
} }
async getPipelineStatus(data) {
this.logger.info('PipelinesClientService - GetPipelineStatus');
const statusPipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
)
.then((res) => {
const statusArray =
res.status?.sort((a, b) => {
if (a.id < b.id) {
return 1;
} else {
return -1;
}
}) || [];
return { status: statusArray };
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
this.logger.info('Done');
return statusPipelineResponse;
}
async runPipeline({ id, info }: IIdRequest) {
this.logger.info('PipelinesClientService - RunPipeline');
const statusPipelineResponse = await lastValueFrom(
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
).catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
if (statusPipelineResponse.status == false) {
throw new ConflictException(
'This pipeline is not ready yet to execute, Try again later!',
);
}
this.logger.info('Done');
return statusPipelineResponse;
}
} }
@@ -20,11 +20,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { import {
Authenticated, Authenticated,
RequireAllPermissions, RequireAllPermissions,
RequireModule,
} from '../../decorators/authentication.decorator'; } from '../../decorators/authentication.decorator';
import { User, RequestUser } from '../../decorators/user.decorator'; import { User, RequestUser } from '../../decorators/user.decorator';
import { PlatformApiService } from './platform-api.service'; import { PlatformApiService } from './platform-api.service';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { ElasticsearchService } from '../../services/elasticsearch'; import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service'; import { CustomersService } from '../customers/customers.service';
@@ -50,7 +49,6 @@ type RenameTablesBody = {
@ApiTags('Platform API') @ApiTags('Platform API')
@Controller('platform') @Controller('platform')
@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT)
export class PlatformApiController { export class PlatformApiController {
private logger: any; private logger: any;
@@ -1,7 +1,6 @@
import { Test, TestingModule } from '@nestjs/testing'; import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteController } from './release_note.controller'; import { ReleaseNoteController } from './release_note.controller';
import { ReleaseNoteService } from './release_note.service'; import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteController', () => { describe('ReleaseNoteController', () => {
let controller: ReleaseNoteController; let controller: ReleaseNoteController;
@@ -9,7 +8,7 @@ describe('ReleaseNoteController', () => {
beforeEach(async () => { beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({ const module: TestingModule = await Test.createTestingModule({
controllers: [ReleaseNoteController], controllers: [ReleaseNoteController],
providers: [ReleaseNoteService, DadosferaLogger], providers: [ReleaseNoteService],
}).compile(); }).compile();
controller = module.get<ReleaseNoteController>(ReleaseNoteController); controller = module.get<ReleaseNoteController>(ReleaseNoteController);
@@ -1,13 +1,12 @@
import { Test, TestingModule } from '@nestjs/testing'; import { Test, TestingModule } from '@nestjs/testing';
import { ReleaseNoteService } from './release_note.service'; import { ReleaseNoteService } from './release_note.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
describe('ReleaseNoteService', () => { describe('ReleaseNoteService', () => {
let service: ReleaseNoteService; let service: ReleaseNoteService;
beforeEach(async () => { beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({ const module: TestingModule = await Test.createTestingModule({
providers: [ReleaseNoteService, DadosferaLogger], providers: [ReleaseNoteService],
}).compile(); }).compile();
service = module.get<ReleaseNoteService>(ReleaseNoteService); service = module.get<ReleaseNoteService>(ReleaseNoteService);
+2 -5
View File
@@ -1,8 +1,5 @@
export interface Info { import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
user_id: string;
customer_id: string;
customer: string;
}
export interface ICreateTransformationsRequest { export interface ICreateTransformationsRequest {
transformations: Transformation[]; transformations: Transformation[];
info: Info; info: Info;
-1
View File
@@ -9,7 +9,6 @@ interface IMetadata {
details?: string; details?: string;
sensitive?: string; sensitive?: string;
roles?: string[]; roles?: string[];
customer_modules?: string[];
is_data_manager?: boolean; is_data_manager?: boolean;
access_token?: string; access_token?: string;
host?: string; host?: string;