mirror of
https://github.com/dadosfera/maestro.git
synced 2026-10-31 19:09:07 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
65ab16236f | ||
|
|
b4cc8151d7 |
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
FROM node:20-alpine AS base_image
|
FROM node:20-alpine AS base_image
|
||||||
RUN npm install -g npm@10.8.2
|
RUN npm install -g npm@latest
|
||||||
|
|
||||||
FROM base_image AS build_base
|
FROM base_image AS build_base
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|||||||
+87
-428
File diff suppressed because it is too large
Load Diff
Generated
+15
-4
@@ -16,7 +16,8 @@
|
|||||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
|
"@dadosfera/protospack-v2": "3.40.0-beta.8",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
@@ -1734,10 +1735,20 @@
|
|||||||
"winston-log2gelf": "^2.4.0"
|
"winston-log2gelf": "^2.4.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/@dadosfera/protospack": {
|
||||||
|
"version": "2.5.3",
|
||||||
|
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
|
||||||
|
"integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
|
||||||
|
"license": "ISC",
|
||||||
|
"dependencies": {
|
||||||
|
"rxjs": "^7.5.5"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@dadosfera/protospack-v2": {
|
"node_modules/@dadosfera/protospack-v2": {
|
||||||
"version": "3.40.0-beta.14",
|
"version": "3.40.0-beta.8",
|
||||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
|
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz",
|
||||||
"integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
|
"integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==",
|
||||||
|
"license": "ISC",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"rxjs": "^7.5.5"
|
"rxjs": "^7.5.5"
|
||||||
|
|||||||
+2
-1
@@ -34,7 +34,8 @@
|
|||||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
|
"@dadosfera/protospack-v2": "3.40.0-beta.8",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
|
|||||||
import { NetworkConfigModule } from './modules/network-config/network-config.module';
|
import { NetworkConfigModule } from './modules/network-config/network-config.module';
|
||||||
import { InputsModule } from './modules/inputs/inputs.module';
|
import { InputsModule } from './modules/inputs/inputs.module';
|
||||||
import { OauthModule } from './modules/oauth/oauth.module';
|
import { OauthModule } from './modules/oauth/oauth.module';
|
||||||
|
import { PipelinesModule } from './modules/pipelines/pipelines.module';
|
||||||
import { TransformationsModule } from './modules/transformations/transformations.module';
|
import { TransformationsModule } from './modules/transformations/transformations.module';
|
||||||
import { HealthModule } from './modules/health/health.module';
|
import { HealthModule } from './modules/health/health.module';
|
||||||
import { CatalogModule } from './modules/catalog/catalog.module';
|
import { CatalogModule } from './modules/catalog/catalog.module';
|
||||||
@@ -59,6 +60,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
|
|||||||
PermissionsModule,
|
PermissionsModule,
|
||||||
TermsOfUseModule,
|
TermsOfUseModule,
|
||||||
ConnectionTestModule,
|
ConnectionTestModule,
|
||||||
|
PipelinesModule,
|
||||||
TransformationsModule,
|
TransformationsModule,
|
||||||
UsersModule,
|
UsersModule,
|
||||||
RolesModule,
|
RolesModule,
|
||||||
|
|||||||
@@ -357,16 +357,6 @@ export const PERMISSIONS_GROUPS = {
|
|||||||
'es-es': 'Crear y editar atributos en el catálogo',
|
'es-es': 'Crear y editar atributos en el catálogo',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
CERTIFY: {
|
|
||||||
seqid: 53,
|
|
||||||
claim: 'catalog:certify',
|
|
||||||
usage: PermissionUsages.PUBLIC,
|
|
||||||
name: {
|
|
||||||
'pt-br': 'Alterar o status de certificação dos Ativos',
|
|
||||||
'en-us': "Change Assets' certification status",
|
|
||||||
'es-es': 'Cambiar el estado de certificación de los Activos',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
DELETE: {
|
DELETE: {
|
||||||
seqid: 1,
|
seqid: 1,
|
||||||
claim: 'catalog:delete',
|
claim: 'catalog:delete',
|
||||||
@@ -722,8 +712,6 @@ export const DADOSFERA_MODULES_KEYS = {
|
|||||||
PII: 'pii',
|
PII: 'pii',
|
||||||
EMBED: 'embedded-analytics',
|
EMBED: 'embedded-analytics',
|
||||||
EMBED_ASSIGNED: 'embed-assigned',
|
EMBED_ASSIGNED: 'embed-assigned',
|
||||||
CATALOG: 'catalog',
|
|
||||||
COLLECT: 'collect',
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
|
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ import {
|
|||||||
HttpStatus,
|
HttpStatus,
|
||||||
Res,
|
Res,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { ValidationPipe } from '../../pipes/object-validation.pipe';
|
|
||||||
import {
|
import {
|
||||||
ApiCreatedResponse,
|
ApiCreatedResponse,
|
||||||
ApiHeaders,
|
ApiHeaders,
|
||||||
@@ -47,7 +46,6 @@ import {
|
|||||||
IMakeAComment,
|
IMakeAComment,
|
||||||
IOneDataAsset,
|
IOneDataAsset,
|
||||||
IPreviewResponse,
|
IPreviewResponse,
|
||||||
IUpdateCertificationStatusRequest,
|
|
||||||
IUpdateDataRequest,
|
IUpdateDataRequest,
|
||||||
TriggerCatalogReq,
|
TriggerCatalogReq,
|
||||||
TriggerCatalogRes,
|
TriggerCatalogRes,
|
||||||
@@ -85,9 +83,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async searchCatalog(
|
async searchCatalog(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Query() query: ICatalogAllRequest,
|
@Query() query: ICatalogAllRequest,
|
||||||
@@ -127,9 +122,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async dowloadAsserts(
|
async dowloadAsserts(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Query() query: ICatalogAllRequest,
|
@Query() query: ICatalogAllRequest,
|
||||||
@@ -173,9 +165,6 @@ export class CatalogController {
|
|||||||
|
|
||||||
@ApiInternalOnlyEndpoint()
|
@ApiInternalOnlyEndpoint()
|
||||||
@Get('data-asset')
|
@Get('data-asset')
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
|
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
|
||||||
const { username, user_id, customer_id, customer_name, permissions } = user;
|
const { username, user_id, customer_id, customer_name, permissions } = user;
|
||||||
const { pipeline, object } = query;
|
const { pipeline, object } = query;
|
||||||
@@ -234,9 +223,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async findAllTags(@Body() body) {
|
async findAllTags(@Body() body) {
|
||||||
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
|
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
|
||||||
user: body.info.user_id,
|
user: body.info.user_id,
|
||||||
@@ -283,31 +269,11 @@ export class CatalogController {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get('custom-properties')
|
|
||||||
@RequireSomePermission(
|
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
|
||||||
)
|
|
||||||
async getCustomPropertyDefinitions(@User() user: RequestUser) {
|
|
||||||
const { customer_id, customer_name, user_id, username } = user;
|
|
||||||
const metadata = PackTheMetadata({
|
|
||||||
customer_id,
|
|
||||||
customer_name,
|
|
||||||
user_id,
|
|
||||||
username,
|
|
||||||
});
|
|
||||||
|
|
||||||
return this.catalogService.getCustomPropertyDefinitions(metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Get('data-asset/:id')
|
@Get('data-asset/:id')
|
||||||
@RequireSomePermission(
|
@RequireSomePermission(
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async getDataAsset(
|
async getDataAsset(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Param('id') id: string,
|
@Param('id') id: string,
|
||||||
@@ -419,9 +385,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async getDataAssetColumnsMetadata(
|
async getDataAssetColumnsMetadata(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
@@ -453,9 +416,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async getDataAssetPreview(
|
async getDataAssetPreview(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
@@ -487,9 +447,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async getDataAssetDocs(
|
async getDataAssetDocs(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
@@ -521,9 +478,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async updateDataAsset(
|
async updateDataAsset(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
@@ -539,8 +493,6 @@ export class CatalogController {
|
|||||||
language,
|
language,
|
||||||
});
|
});
|
||||||
|
|
||||||
delete (body as any).certification_status;
|
|
||||||
|
|
||||||
const result = await this.catalogService.updateOneDataAsset({
|
const result = await this.catalogService.updateOneDataAsset({
|
||||||
body,
|
body,
|
||||||
data_asset_id,
|
data_asset_id,
|
||||||
@@ -554,44 +506,11 @@ export class CatalogController {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Put('data-asset/:id/certification-status')
|
|
||||||
@RequireSomePermission(
|
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.CERTIFY,
|
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
|
||||||
)
|
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async updateDataAssetCertificationStatus(
|
|
||||||
@User() user: RequestUser,
|
|
||||||
@Language() language: LanguageEnum,
|
|
||||||
@Param('id') data_asset_id: string,
|
|
||||||
@Body(new ValidationPipe()) body: IUpdateCertificationStatusRequest,
|
|
||||||
): Promise<IUpdateCertificationStatusRequest> {
|
|
||||||
const { customer_id, customer_name, user_id, username } = user;
|
|
||||||
const metadata = PackTheMetadata({
|
|
||||||
customer_id,
|
|
||||||
customer_name,
|
|
||||||
user_id,
|
|
||||||
username,
|
|
||||||
language,
|
|
||||||
});
|
|
||||||
|
|
||||||
return this.catalogService.updateCertificationStatus({
|
|
||||||
body,
|
|
||||||
data_asset_id,
|
|
||||||
metadata,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post('data-asset/:id/docs')
|
@Post('data-asset/:id/docs')
|
||||||
@RequireSomePermission(
|
@RequireSomePermission(
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async manageDataAssetDocs(
|
async manageDataAssetDocs(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Headers() headers,
|
@Headers() headers,
|
||||||
@@ -629,9 +548,6 @@ export class CatalogController {
|
|||||||
|
|
||||||
@ApiInternalOnlyEndpoint()
|
@ApiInternalOnlyEndpoint()
|
||||||
@Put('data-asset/:id/manage-permissions')
|
@Put('data-asset/:id/manage-permissions')
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async manageDataAssetPermissions(
|
async manageDataAssetPermissions(
|
||||||
@Param('id') id: string,
|
@Param('id') id: string,
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@@ -654,9 +570,6 @@ export class CatalogController {
|
|||||||
|
|
||||||
@ApiInternalOnlyEndpoint()
|
@ApiInternalOnlyEndpoint()
|
||||||
@Put('data-asset/:id/revoke-permissions')
|
@Put('data-asset/:id/revoke-permissions')
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async revokeDataAssetPermissions(
|
async revokeDataAssetPermissions(
|
||||||
@Param('id') id: string,
|
@Param('id') id: string,
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@@ -682,9 +595,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async createDataAsset(
|
async createDataAsset(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Body() body: ICreateDataAsset,
|
@Body() body: ICreateDataAsset,
|
||||||
@@ -709,9 +619,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async commentOnDataAsset(
|
async commentOnDataAsset(
|
||||||
@Param('id') id: string,
|
@Param('id') id: string,
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@@ -738,9 +645,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
|
async deleteDataAsset(@Param('id') id: string, @User() user: RequestUser) {
|
||||||
const { customer_id, customer_name, user_id, username } = user;
|
const { customer_id, customer_name, user_id, username } = user;
|
||||||
const metadata = PackTheMetadata({
|
const metadata = PackTheMetadata({
|
||||||
@@ -762,9 +666,6 @@ export class CatalogController {
|
|||||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async deleteComment(
|
async deleteComment(
|
||||||
@Param('id') id: string,
|
@Param('id') id: string,
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@@ -918,9 +819,6 @@ export class CatalogController {
|
|||||||
|
|
||||||
@Get('nimbus-dashboards')
|
@Get('nimbus-dashboards')
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
|
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.CATALOG
|
|
||||||
)
|
|
||||||
async getNimbusDashboards(
|
async getNimbusDashboards(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@Body() body: GetNimbusDashboardsRequest,
|
@Body() body: GetNimbusDashboardsRequest,
|
||||||
|
|||||||
@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|||||||
import { CatalogController } from './catalog.controller';
|
import { CatalogController } from './catalog.controller';
|
||||||
import { CatalogClientConfiguration } from './catalog-client';
|
import { CatalogClientConfiguration } from './catalog-client';
|
||||||
import { ClientsModule } from '@nestjs/microservices';
|
import { ClientsModule } from '@nestjs/microservices';
|
||||||
|
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||||
import { UsersModule } from '../users/users.module';
|
import { UsersModule } from '../users/users.module';
|
||||||
import { RolesModule } from '../roles/roles.module';
|
import { RolesModule } from '../roles/roles.module';
|
||||||
import { CustomersModule } from '../customers/customers.module';
|
import { CustomersModule } from '../customers/customers.module';
|
||||||
import { ShareModule } from './share/share.module';
|
import { ShareModule } from './share/share.module';
|
||||||
import { CatalogService } from './catalog.service';
|
import { CatalogService } from './catalog.service';
|
||||||
|
import { MixpanelModule } from '../mixpanel/mixpanel.module';
|
||||||
|
|
||||||
const client = new CatalogClientConfiguration();
|
const client = new CatalogClientConfiguration();
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
ClientsModule.register([client.providerOptions]),
|
ClientsModule.register([client.providerOptions]),
|
||||||
|
OldPipelineModule,
|
||||||
UsersModule,
|
UsersModule,
|
||||||
RolesModule,
|
RolesModule,
|
||||||
CustomersModule,
|
CustomersModule,
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ import {
|
|||||||
AssetReporter,
|
AssetReporter,
|
||||||
BatchRemoveRlsRulesRequest,
|
BatchRemoveRlsRulesRequest,
|
||||||
CreateDataDocsDTO,
|
CreateDataDocsDTO,
|
||||||
IUpdateCertificationStatusRequest,
|
|
||||||
IUpdateDataRequest,
|
IUpdateDataRequest,
|
||||||
TriggerCatalogReq,
|
TriggerCatalogReq,
|
||||||
} from './dtos';
|
} from './dtos';
|
||||||
@@ -120,10 +119,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async getCustomPropertyDefinitions(metadata: Metadata) {
|
|
||||||
return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata));
|
|
||||||
}
|
|
||||||
|
|
||||||
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
|
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
|
||||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||||
if (!data.embed) data.embed = undefined;
|
if (!data.embed) data.embed = undefined;
|
||||||
@@ -389,28 +384,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return { data_asset: asset[0] };
|
return { data_asset: asset[0] };
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateCertificationStatus(data: {
|
|
||||||
data_asset_id: string;
|
|
||||||
body: IUpdateCertificationStatusRequest;
|
|
||||||
metadata: Metadata;
|
|
||||||
}) {
|
|
||||||
const { body, data_asset_id, metadata } = data;
|
|
||||||
|
|
||||||
await lastValueFrom(
|
|
||||||
this.catalogWriteService.UpdateDataAsset(
|
|
||||||
{
|
|
||||||
id: data_asset_id,
|
|
||||||
changes: JSON.stringify({
|
|
||||||
certification_status: body.certification_status,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
metadata,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
|
|
||||||
return { certification_status: body.certification_status };
|
|
||||||
}
|
|
||||||
|
|
||||||
async updateOneDataAsset(data: {
|
async updateOneDataAsset(data: {
|
||||||
data_asset_id: string;
|
data_asset_id: string;
|
||||||
customer_id: string;
|
customer_id: string;
|
||||||
|
|||||||
@@ -1,10 +1,4 @@
|
|||||||
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
|
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
|
||||||
import {
|
|
||||||
IsEnum,
|
|
||||||
IsNotEmpty,
|
|
||||||
IsOptional,
|
|
||||||
IsString,
|
|
||||||
} from 'class-validator';
|
|
||||||
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||||
|
|
||||||
export enum DataAssetShareType {
|
export enum DataAssetShareType {
|
||||||
@@ -12,12 +6,6 @@ export enum DataAssetShareType {
|
|||||||
public = 'public',
|
public = 'public',
|
||||||
private = 'private',
|
private = 'private',
|
||||||
}
|
}
|
||||||
export enum CertificationStatus {
|
|
||||||
draft = 'draft',
|
|
||||||
in_review = 'in_review',
|
|
||||||
approved = 'approved',
|
|
||||||
deprecated = 'deprecated',
|
|
||||||
}
|
|
||||||
export enum OrderEnum {
|
export enum OrderEnum {
|
||||||
asc = 'asc',
|
asc = 'asc',
|
||||||
desc = 'desc',
|
desc = 'desc',
|
||||||
@@ -203,27 +191,6 @@ export class IData {
|
|||||||
day_opening: number;
|
day_opening: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export enum CustomPropertyType {
|
|
||||||
TEXT = 'text',
|
|
||||||
NUMBER = 'number',
|
|
||||||
DATE = 'date',
|
|
||||||
BOOLEAN = 'boolean',
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CustomPropertyDto {
|
|
||||||
@ApiProperty()
|
|
||||||
key: string;
|
|
||||||
@ApiProperty()
|
|
||||||
value: string;
|
|
||||||
@ApiProperty({ enum: CustomPropertyType })
|
|
||||||
type: CustomPropertyType;
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
color?: string;
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
emoji?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class IUpdateDataRequest {
|
export class IUpdateDataRequest {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
name: string;
|
name: string;
|
||||||
@@ -237,16 +204,7 @@ export class IUpdateDataRequest {
|
|||||||
share_type?: DataAssetShareType;
|
share_type?: DataAssetShareType;
|
||||||
@ApiPropertyOptional()
|
@ApiPropertyOptional()
|
||||||
docs?: string;
|
docs?: string;
|
||||||
@ApiPropertyOptional({ type: [CustomPropertyDto] })
|
|
||||||
custom_properties?: CustomPropertyDto[];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export class IUpdateCertificationStatusRequest {
|
|
||||||
@ApiProperty({ enum: CertificationStatus })
|
|
||||||
@IsEnum(CertificationStatus)
|
|
||||||
certification_status: CertificationStatus;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ICreateDataAsset implements CreateDataAssetRequest {
|
export class ICreateDataAsset implements CreateDataAssetRequest {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
display_name: string;
|
display_name: string;
|
||||||
|
|||||||
@@ -24,19 +24,15 @@ import {
|
|||||||
GetTableMetadataReq,
|
GetTableMetadataReq,
|
||||||
} from './dto/connection-test';
|
} from './dto/connection-test';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { Authenticated, RequireModule } from 'src/decorators/authentication.decorator';
|
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||||
import { DADOSFERA_MODULES_KEYS } from 'src/authentication/permissions.enum';
|
|
||||||
|
|
||||||
@ApiInternalOnlyController()
|
@ApiInternalOnlyController()
|
||||||
@ApiTags('Connection Test')
|
@ApiTags('Connection Test')
|
||||||
@Controller('connection-test')
|
@Controller('connection-test')
|
||||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||||
@Authenticated()
|
@Authenticated()
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
export class ConnectionTestController {
|
export class ConnectionTestController {
|
||||||
logger: any;
|
logger: any;
|
||||||
constructor(
|
constructor(
|
||||||
|
|||||||
@@ -16,9 +16,8 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|||||||
import {
|
import {
|
||||||
Authenticated,
|
Authenticated,
|
||||||
RequireAllPermissions,
|
RequireAllPermissions,
|
||||||
RequireModule,
|
|
||||||
} from 'src/decorators/authentication.decorator';
|
} from 'src/decorators/authentication.decorator';
|
||||||
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||||
import { ValidationPipe } from '../../pipes/object-validation.pipe';
|
import { ValidationPipe } from '../../pipes/object-validation.pipe';
|
||||||
import {
|
import {
|
||||||
@@ -40,9 +39,6 @@ const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
|
|||||||
@ApiTags('connections')
|
@ApiTags('connections')
|
||||||
@Authenticated()
|
@Authenticated()
|
||||||
@Controller('connections')
|
@Controller('connections')
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
export class ConnectionController {
|
export class ConnectionController {
|
||||||
logger: any;
|
logger: any;
|
||||||
constructor(
|
constructor(
|
||||||
|
|||||||
@@ -25,10 +25,9 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|||||||
import {
|
import {
|
||||||
Authenticated,
|
Authenticated,
|
||||||
RequireAllPermissions,
|
RequireAllPermissions,
|
||||||
RequireModule,
|
|
||||||
RequireSomePermission,
|
RequireSomePermission,
|
||||||
} from 'src/decorators/authentication.decorator';
|
} from 'src/decorators/authentication.decorator';
|
||||||
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||||
import { Language } from 'src/decorators/language.decorator';
|
import { Language } from 'src/decorators/language.decorator';
|
||||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||||
@@ -100,9 +99,6 @@ export class ConnectorController {
|
|||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async getAllConnectors(
|
async getAllConnectors(
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
@Query() queries: GetAllDto,
|
@Query() queries: GetAllDto,
|
||||||
@@ -135,9 +131,6 @@ export class ConnectorController {
|
|||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async getConnectorsTags() {
|
async getConnectorsTags() {
|
||||||
return await this.connectorClientService.getConnectorsTags();
|
return await this.connectorClientService.getConnectorsTags();
|
||||||
}
|
}
|
||||||
@@ -150,9 +143,6 @@ export class ConnectorController {
|
|||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async getConnector(
|
async getConnector(
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
@Param('plugin') plugin: string,
|
@Param('plugin') plugin: string,
|
||||||
@@ -181,9 +171,6 @@ export class ConnectorController {
|
|||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE,
|
||||||
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
|
||||||
)
|
)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async getConnectorDetails(
|
async getConnectorDetails(
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
@Param('plugin') plugin: string,
|
@Param('plugin') plugin: string,
|
||||||
@@ -206,9 +193,6 @@ export class ConnectorController {
|
|||||||
@Put('/:plugin')
|
@Put('/:plugin')
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
|
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
|
||||||
@ApiConsumes('multipart/form-data')
|
@ApiConsumes('multipart/form-data')
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async updateConnector(
|
async updateConnector(
|
||||||
@Param('plugin') plugin: string,
|
@Param('plugin') plugin: string,
|
||||||
@Body() body: UpdateDto,
|
@Body() body: UpdateDto,
|
||||||
@@ -230,9 +214,6 @@ export class ConnectorController {
|
|||||||
|
|
||||||
@Put('/:plugin/add-tag')
|
@Put('/:plugin/add-tag')
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
|
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async addTagOnConnector(
|
async addTagOnConnector(
|
||||||
@Param('plugin') plugin: string,
|
@Param('plugin') plugin: string,
|
||||||
@Body() body: AddTagDto,
|
@Body() body: AddTagDto,
|
||||||
@@ -260,9 +241,6 @@ export class ConnectorController {
|
|||||||
|
|
||||||
@Put('/:plugin/remove-tag')
|
@Put('/:plugin/remove-tag')
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
|
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.UPDATE)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async removeTagOnConnector(
|
async removeTagOnConnector(
|
||||||
@Param('plugin') plugin: string,
|
@Param('plugin') plugin: string,
|
||||||
@Body() body: RemoveTagDto,
|
@Body() body: RemoveTagDto,
|
||||||
@@ -291,9 +269,6 @@ export class ConnectorController {
|
|||||||
|
|
||||||
@Delete('/:plugin')
|
@Delete('/:plugin')
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
|
@RequireAllPermissions(PERMISSIONS_GROUPS.CONNECTORS.permissions.DELETE)
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
async deleteConnector(
|
async deleteConnector(
|
||||||
@Param('plugin') plugin: string,
|
@Param('plugin') plugin: string,
|
||||||
@Query('version') version: string,
|
@Query('version') version: string,
|
||||||
|
|||||||
@@ -1,8 +1,4 @@
|
|||||||
export interface Info {
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
user_id: string;
|
|
||||||
customer_id: string;
|
|
||||||
customer: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Values {
|
interface Values {
|
||||||
jdbc_user: string;
|
jdbc_user: string;
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ import {
|
|||||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||||
import { CreateInputReq } from './dtos/input.model';
|
import { CreateInputReq } from './dtos/input.model';
|
||||||
import { Metadata } from '@grpc/grpc-js';
|
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
|
|
||||||
@@ -74,10 +73,10 @@ export class InputsService {
|
|||||||
objectCamelToSnake(createInputResponse);
|
objectCamelToSnake(createInputResponse);
|
||||||
return createInputResponse;
|
return createInputResponse;
|
||||||
},
|
},
|
||||||
update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
|
update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
|
||||||
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
|
this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
|
||||||
const updateInputResponse = await lastValueFrom(
|
const updateInputResponse = await lastValueFrom(
|
||||||
this.inputWriteService.InputUpdate(updateInputDTO, metadata),
|
this.inputWriteService.InputUpdate(updateInputDTO),
|
||||||
);
|
);
|
||||||
|
|
||||||
return updateInputResponse;
|
return updateInputResponse;
|
||||||
@@ -207,7 +206,7 @@ export class InputsService {
|
|||||||
return findOneInputResponse;
|
return findOneInputResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
async update(id: string, data, info: Info, metadata?: Metadata) {
|
async update(id: string, data, info: Info) {
|
||||||
// this.validateCron({ ...data, info });
|
// this.validateCron({ ...data, info });
|
||||||
try {
|
try {
|
||||||
const {
|
const {
|
||||||
@@ -218,7 +217,7 @@ export class InputsService {
|
|||||||
id,
|
id,
|
||||||
...data,
|
...data,
|
||||||
info,
|
info,
|
||||||
}, metadata);
|
});
|
||||||
|
|
||||||
const updateInputResponse = this.adjustInputPayload(
|
const updateInputResponse = this.adjustInputPayload(
|
||||||
input,
|
input,
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
|
||||||
|
import { ClientGrpc } from '@nestjs/microservices';
|
||||||
|
import {
|
||||||
|
PipelineServicesNames,
|
||||||
|
PipelinesServiceInterface,
|
||||||
|
} from '@dadosfera/protospack';
|
||||||
|
import { lastValueFrom } from 'rxjs';
|
||||||
|
|
||||||
|
import { IIdRequest } from './interfaces';
|
||||||
|
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
|
|
||||||
|
export class PipelinesClientService implements OnModuleInit {
|
||||||
|
private pipelineService: PipelinesServiceInterface;
|
||||||
|
logger: DadosferaLogger;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
@Inject(DadosferaLogger)
|
||||||
|
dadosferaLogger: DadosferaLogger,
|
||||||
|
@Inject(PipelinesClientConfiguration.name)
|
||||||
|
private readonly grpcClient: ClientGrpc,
|
||||||
|
) {
|
||||||
|
this.logger = dadosferaLogger.logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
onModuleInit() {
|
||||||
|
this.pipelineService =
|
||||||
|
this.grpcClient.getService<PipelinesServiceInterface>(
|
||||||
|
PipelineServicesNames.PipelineService,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getPipelineStatus(data) {
|
||||||
|
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
||||||
|
|
||||||
|
const statusPipelineResponse = await lastValueFrom(
|
||||||
|
this.pipelineService.getPipelineStatus(data),
|
||||||
|
)
|
||||||
|
.then((res) => {
|
||||||
|
const statusArray =
|
||||||
|
res.status?.sort((a, b) => {
|
||||||
|
if (a.id < b.id) {
|
||||||
|
return 1;
|
||||||
|
} else {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}) || [];
|
||||||
|
return { status: statusArray };
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
this.logger.error(err.message);
|
||||||
|
throw new Error(err);
|
||||||
|
});
|
||||||
|
this.logger.info('Done');
|
||||||
|
|
||||||
|
return statusPipelineResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
async runPipeline({ id, info }: IIdRequest) {
|
||||||
|
this.logger.info('PipelinesClientService - RunPipeline');
|
||||||
|
const statusPipelineResponse = await lastValueFrom(
|
||||||
|
this.pipelineService.triggerPipeline({ id, info }),
|
||||||
|
).catch((err) => {
|
||||||
|
this.logger.error(err.message);
|
||||||
|
throw new Error(err);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (statusPipelineResponse.status == false) {
|
||||||
|
throw new ConflictException(
|
||||||
|
'This pipeline is not ready yet to execute, Try again later!',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.info('Done');
|
||||||
|
return statusPipelineResponse;
|
||||||
|
}
|
||||||
|
}
|
||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
|
|
||||||
|
export interface ICreatePipelineDto {
|
||||||
|
input: IdRequest;
|
||||||
|
transformations: IdRequest[];
|
||||||
|
output: IdRequest;
|
||||||
|
tags: string[];
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
info: Info;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IdRequest {
|
||||||
|
id: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IIdRequest {
|
||||||
|
id: string;
|
||||||
|
info: Info;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IUpdatePipelineRequest {
|
||||||
|
input: IdRequest;
|
||||||
|
transformations: IdRequest[];
|
||||||
|
output: IdRequest;
|
||||||
|
tags: string[];
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
id: string;
|
||||||
|
info: Info;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface IGetPipelineLogsRequest {
|
||||||
|
id: string;
|
||||||
|
details: string;
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import {
|
||||||
|
ClientsProviderAsyncOptions,
|
||||||
|
GrpcOptions,
|
||||||
|
Transport,
|
||||||
|
} from '@nestjs/microservices';
|
||||||
|
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
|
||||||
|
import { credentials } from '@grpc/grpc-js';
|
||||||
|
|
||||||
|
const isLocalConnection =
|
||||||
|
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
||||||
|
process.env.PIFACTORY_URL.includes('0.0.0.0');
|
||||||
|
|
||||||
|
export class PipelinesClientConfiguration {
|
||||||
|
public name = 'PipelinesClientConfiguration';
|
||||||
|
private config: GrpcOptions = {
|
||||||
|
transport: Transport.GRPC,
|
||||||
|
options: {
|
||||||
|
url: process.env.PIFACTORY_URL,
|
||||||
|
package: PipelinePackages,
|
||||||
|
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
||||||
|
protoPath: PipelineProtoFilePath,
|
||||||
|
loader: {
|
||||||
|
keepCase: true,
|
||||||
|
enums: String,
|
||||||
|
defaults: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
providerOptions: ClientsProviderAsyncOptions = {
|
||||||
|
name: this.name,
|
||||||
|
...this.config,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
|
||||||
|
import { ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||||
|
import {
|
||||||
|
AuthenticateCondition,
|
||||||
|
Authenticated,
|
||||||
|
RequireSomePermission,
|
||||||
|
} from 'src/decorators/authentication.decorator';
|
||||||
|
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||||
|
import { PipelinesService } from './pipelines.service';
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||||
|
|
||||||
|
@ApiInternalOnlyController()
|
||||||
|
@ApiTags('Pipelines')
|
||||||
|
@Controller('pipelines')
|
||||||
|
@Authenticated()
|
||||||
|
export class PipelinesController {
|
||||||
|
logger: DadosferaLogger;
|
||||||
|
constructor(
|
||||||
|
@Inject(DadosferaLogger)
|
||||||
|
dadosferaLogger: DadosferaLogger,
|
||||||
|
private pipelineService: PipelinesService,
|
||||||
|
) {
|
||||||
|
this.logger = dadosferaLogger.logger;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Post('start/:id')
|
||||||
|
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||||
|
@ApiOperation({
|
||||||
|
deprecated: true,
|
||||||
|
description:
|
||||||
|
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
|
||||||
|
})
|
||||||
|
async activate(@Param('id') id: string, @Body() body) {
|
||||||
|
const { info } = body;
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
|
||||||
|
{
|
||||||
|
user: body.info.user_id,
|
||||||
|
customer: body.info.customer,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const response = await this.pipelineService.runPipeline({ id, info });
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get(':id/status')
|
||||||
|
@ApiOperation({
|
||||||
|
deprecated: true,
|
||||||
|
description:
|
||||||
|
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
|
||||||
|
})
|
||||||
|
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||||
|
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||||
|
body.id = id;
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
|
||||||
|
{
|
||||||
|
user: body.info.user_id,
|
||||||
|
customer: body.info.customer,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const response = await this.pipelineService.getPipelineStatus(body);
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { ClientsModule } from '@nestjs/microservices';
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
|
import { PipelinesController } from './pipelines.controller';
|
||||||
|
import { PipelinesService } from './pipelines.service';
|
||||||
|
|
||||||
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
|
import { PipelinesClientService } from './client.service';
|
||||||
|
|
||||||
|
const client = new PipelinesClientConfiguration();
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [ClientsModule.register([client.providerOptions])],
|
||||||
|
controllers: [PipelinesController],
|
||||||
|
providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
|
||||||
|
exports: [PipelinesService],
|
||||||
|
})
|
||||||
|
export class PipelinesModule {}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
|
||||||
|
import { PipelinesClientService } from './client.service';
|
||||||
|
import { IIdRequest } from './interfaces';
|
||||||
|
import { objectCamelToSnake } from 'src/utils/CaseConverter';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class PipelinesService {
|
||||||
|
constructor(private pipelineClient: PipelinesClientService) {}
|
||||||
|
|
||||||
|
async getPipelineStatus(data: IIdRequest) {
|
||||||
|
try {
|
||||||
|
const pipelineStatusResponse =
|
||||||
|
await this.pipelineClient.getPipelineStatus(data);
|
||||||
|
|
||||||
|
return objectCamelToSnake(pipelineStatusResponse);
|
||||||
|
} catch (err) {
|
||||||
|
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async runPipeline({ id, info }: IIdRequest) {
|
||||||
|
try {
|
||||||
|
const triggerPipelineResponse = await this.pipelineClient.runPipeline({
|
||||||
|
id,
|
||||||
|
info,
|
||||||
|
});
|
||||||
|
|
||||||
|
return objectCamelToSnake(triggerPipelineResponse);
|
||||||
|
} catch (err) {
|
||||||
|
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||||
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
|
|
||||||
export class PipelineInputsDTO {
|
export class PipelineInputsDTO {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
@@ -61,12 +62,6 @@ export interface IIdRequest {
|
|||||||
info: Info;
|
info: Info;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface Info {
|
|
||||||
user_id: string;
|
|
||||||
customer_id: string;
|
|
||||||
customer: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IUpdatePipelineRequest {
|
export interface IUpdatePipelineRequest {
|
||||||
input: IdRequest;
|
input: IdRequest;
|
||||||
transformations: IdRequest[];
|
transformations: IdRequest[];
|
||||||
|
|||||||
@@ -25,16 +25,16 @@ import {
|
|||||||
} from '@nestjs/swagger';
|
} from '@nestjs/swagger';
|
||||||
import {
|
import {
|
||||||
RequireAllPermissions,
|
RequireAllPermissions,
|
||||||
RequireModule,
|
|
||||||
RequireSomePermission,
|
RequireSomePermission,
|
||||||
} from 'src/decorators/authentication.decorator';
|
} from 'src/decorators/authentication.decorator';
|
||||||
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||||
import { PipelinesService } from './pipelines.service';
|
import { PipelinesService } from './pipelines.service';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
|
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
|
||||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||||
|
|
||||||
|
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
|
||||||
import {
|
import {
|
||||||
ICompleteUploadCSVFile,
|
ICompleteUploadCSVFile,
|
||||||
ICreatePipelineCSVFile,
|
ICreatePipelineCSVFile,
|
||||||
@@ -58,15 +58,14 @@ type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
|
|||||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||||
@Controller('pipelinesV2')
|
@Controller('pipelinesV2')
|
||||||
@RequireModule(
|
|
||||||
DADOSFERA_MODULES_KEYS.COLLECT
|
|
||||||
)
|
|
||||||
export class PipelinesController {
|
export class PipelinesController {
|
||||||
logger: DadosferaLogger;
|
logger: DadosferaLogger;
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DadosferaLogger)
|
@Inject(DadosferaLogger)
|
||||||
dadosferaLogger: DadosferaLogger,
|
dadosferaLogger: DadosferaLogger,
|
||||||
|
|
||||||
private pipelinesClientService: PipelinesService,
|
private pipelinesClientService: PipelinesService,
|
||||||
|
private oldPipelinesService: OldPipelineService,
|
||||||
) {
|
) {
|
||||||
this.logger = dadosferaLogger.logger;
|
this.logger = dadosferaLogger.logger;
|
||||||
}
|
}
|
||||||
@@ -202,7 +201,7 @@ export class PipelinesController {
|
|||||||
customer: body.info.customer,
|
customer: body.info.customer,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = await this.pipelinesClientService.getPipelineStatus(body);
|
const response = await this.oldPipelinesService.getPipelineStatus(body);
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
@@ -321,6 +320,7 @@ export class PipelinesController {
|
|||||||
) {
|
) {
|
||||||
this.logger.info('PipelinesController - update', { user });
|
this.logger.info('PipelinesController - update', { user });
|
||||||
|
|
||||||
|
const { customer_id, customer_name, user_id, username } = user;
|
||||||
const info: Info = {
|
const info: Info = {
|
||||||
user_id: user.user_id,
|
user_id: user.user_id,
|
||||||
customer: user.customer_name,
|
customer: user.customer_name,
|
||||||
@@ -328,7 +328,13 @@ export class PipelinesController {
|
|||||||
pipeline_id: pipelineId
|
pipeline_id: pipelineId
|
||||||
};
|
};
|
||||||
|
|
||||||
const metadata = PackTheMetadata(user);
|
const metadata = PackTheMetadata({
|
||||||
|
customer_id,
|
||||||
|
customer_name,
|
||||||
|
user_id,
|
||||||
|
username,
|
||||||
|
language,
|
||||||
|
});
|
||||||
|
|
||||||
const response = await this.pipelinesClientService.updatePipelineInput(
|
const response = await this.pipelinesClientService.updatePipelineInput(
|
||||||
pipelineId,
|
pipelineId,
|
||||||
@@ -362,21 +368,6 @@ export class PipelinesController {
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Patch('/:id/upgrade')
|
|
||||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
@HttpCode(HttpStatus.NO_CONTENT)
|
|
||||||
async upgradeConnector(
|
|
||||||
@Language() language: LanguageEnum,
|
|
||||||
@Param('id') id: string,
|
|
||||||
@User() user: RequestUser
|
|
||||||
) {
|
|
||||||
this.logger.info('PipelinesController - upgrade connector');
|
|
||||||
|
|
||||||
const metadata = PackTheMetadata(user);
|
|
||||||
|
|
||||||
await this.pipelinesClientService.upgrade(id, metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Delete(':id')
|
@Delete(':id')
|
||||||
@ApiNoContentResponse()
|
@ApiNoContentResponse()
|
||||||
@HttpCode(HttpStatus.NO_CONTENT)
|
@HttpCode(HttpStatus.NO_CONTENT)
|
||||||
@@ -506,7 +497,7 @@ export class PipelinesController {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
const response = await this.pipelinesClientService.runPipeline({ id, info });
|
const response = await this.oldPipelinesService.runPipeline({ id, info });
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { PipelinesService } from './pipelines.service';
|
|||||||
|
|
||||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
|
|
||||||
|
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||||
import { ConnectorModule } from '../connector/connector.module';
|
import { ConnectorModule } from '../connector/connector.module';
|
||||||
import { InputsModule } from '../inputs/inputs.module';
|
import { InputsModule } from '../inputs/inputs.module';
|
||||||
import { TransformationsModule } from '../transformations/transformations.module';
|
import { TransformationsModule } from '../transformations/transformations.module';
|
||||||
@@ -20,6 +21,7 @@ const client = new PipelinesClientConfiguration();
|
|||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
ClientsModule.register([client.providerOptions]),
|
ClientsModule.register([client.providerOptions]),
|
||||||
|
OldPipelineModule,
|
||||||
ConnectorModule,
|
ConnectorModule,
|
||||||
InputsModule,
|
InputsModule,
|
||||||
TransformationsModule,
|
TransformationsModule,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
/* eslint-disable no-async-promise-executor */
|
/* eslint-disable no-async-promise-executor */
|
||||||
import {
|
import {
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
ConflictException,
|
|
||||||
HttpException,
|
HttpException,
|
||||||
HttpStatus,
|
HttpStatus,
|
||||||
Inject,
|
Inject,
|
||||||
@@ -18,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
|
|||||||
|
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||||
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
||||||
import { Metadata } from '@grpc/grpc-js';
|
import { Metadata } from '@grpc/grpc-js';
|
||||||
import { ConnectorClientService } from '../connector/client.service';
|
import { ConnectorClientService } from '../connector/client.service';
|
||||||
@@ -177,15 +176,6 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
return updatePipelineResponse;
|
return updatePipelineResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
async upgrade(id: string, metadata: Metadata) {
|
|
||||||
await lastValueFrom(
|
|
||||||
this.pipelineWriteService.Upgrade(
|
|
||||||
{ id },
|
|
||||||
metadata,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
|
async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
|
||||||
const { id, metadata, user } = data;
|
const { id, metadata, user } = data;
|
||||||
const info = {
|
const info = {
|
||||||
@@ -383,8 +373,7 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
const updateInputResponse = await this.inputsService.update(
|
const updateInputResponse = await this.inputsService.update(
|
||||||
inputId,
|
inputId,
|
||||||
updateInputDTO,
|
updateInputDTO,
|
||||||
info,
|
info
|
||||||
metadata
|
|
||||||
);
|
);
|
||||||
|
|
||||||
const inputRollback = () => {
|
const inputRollback = () => {
|
||||||
@@ -405,36 +394,34 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
|
|
||||||
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
|
const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
|
||||||
|
|
||||||
if (user.customer_modules.includes('catalog')) {
|
nimbusUpdates.forEach(update => {
|
||||||
nimbusUpdates.forEach(update => {
|
const nimbusRollback = () => {
|
||||||
const nimbusRollback = () => {
|
return this.nimbusService.renameTable(
|
||||||
return this.nimbusService.renameTable(
|
info.customer,
|
||||||
info.customer,
|
update.database,
|
||||||
update.database,
|
{
|
||||||
{
|
table_name: update.table_name,
|
||||||
table_name: update.table_name,
|
table_schema: update.table_schema
|
||||||
table_schema: update.table_schema
|
},
|
||||||
},
|
{
|
||||||
{
|
table_name: update.old_table_name,
|
||||||
table_name: update.old_table_name,
|
table_schema: update.old_table_schema
|
||||||
table_schema: update.old_table_schema
|
}
|
||||||
}
|
);
|
||||||
);
|
|
||||||
}
|
|
||||||
rollback.push(nimbusRollback);
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.updateNimbus(info.customer, nimbusUpdates);
|
|
||||||
} catch (error) {
|
|
||||||
this.logger.error(error);
|
|
||||||
if (error instanceof AxiosError) {
|
|
||||||
this.logger.error(JSON.stringify(error.response.data));
|
|
||||||
}
|
|
||||||
await this.executeRenameRollback(rollback);
|
|
||||||
|
|
||||||
throw new Error("Error Nimbus updating tables");
|
|
||||||
}
|
}
|
||||||
|
rollback.push(nimbusRollback);
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.updateNimbus(info.customer, nimbusUpdates);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(error);
|
||||||
|
if (error instanceof AxiosError) {
|
||||||
|
this.logger.error(JSON.stringify(error.response.data));
|
||||||
|
}
|
||||||
|
await this.executeRenameRollback(rollback);
|
||||||
|
|
||||||
|
throw new Error("Error Nimbus updating tables");
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -636,49 +623,4 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
return assets;
|
return assets;
|
||||||
}
|
}
|
||||||
|
|
||||||
async getPipelineStatus(data) {
|
|
||||||
this.logger.info('PipelinesClientService - GetPipelineStatus');
|
|
||||||
|
|
||||||
const statusPipelineResponse = await lastValueFrom(
|
|
||||||
this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
|
|
||||||
)
|
|
||||||
.then((res) => {
|
|
||||||
const statusArray =
|
|
||||||
res.status?.sort((a, b) => {
|
|
||||||
if (a.id < b.id) {
|
|
||||||
return 1;
|
|
||||||
} else {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}) || [];
|
|
||||||
return { status: statusArray };
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
this.logger.error(err.message);
|
|
||||||
throw new Error(err);
|
|
||||||
});
|
|
||||||
this.logger.info('Done');
|
|
||||||
|
|
||||||
return statusPipelineResponse;
|
|
||||||
}
|
|
||||||
|
|
||||||
async runPipeline({ id, info }: IIdRequest) {
|
|
||||||
this.logger.info('PipelinesClientService - RunPipeline');
|
|
||||||
const statusPipelineResponse = await lastValueFrom(
|
|
||||||
this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
|
|
||||||
).catch((err) => {
|
|
||||||
this.logger.error(err.message);
|
|
||||||
throw new Error(err);
|
|
||||||
});
|
|
||||||
|
|
||||||
if (statusPipelineResponse.status == false) {
|
|
||||||
throw new ConflictException(
|
|
||||||
'This pipeline is not ready yet to execute, Try again later!',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.logger.info('Done');
|
|
||||||
return statusPipelineResponse;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,11 +20,10 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|||||||
import {
|
import {
|
||||||
Authenticated,
|
Authenticated,
|
||||||
RequireAllPermissions,
|
RequireAllPermissions,
|
||||||
RequireModule,
|
|
||||||
} from '../../decorators/authentication.decorator';
|
} from '../../decorators/authentication.decorator';
|
||||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||||
import { PlatformApiService } from './platform-api.service';
|
import { PlatformApiService } from './platform-api.service';
|
||||||
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||||
import { ElasticsearchService } from '../../services/elasticsearch';
|
import { ElasticsearchService } from '../../services/elasticsearch';
|
||||||
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
|
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
|
||||||
import { CustomersService } from '../customers/customers.service';
|
import { CustomersService } from '../customers/customers.service';
|
||||||
@@ -50,7 +49,6 @@ type RenameTablesBody = {
|
|||||||
|
|
||||||
@ApiTags('Platform API')
|
@ApiTags('Platform API')
|
||||||
@Controller('platform')
|
@Controller('platform')
|
||||||
@RequireModule(DADOSFERA_MODULES_KEYS.COLLECT)
|
|
||||||
export class PlatformApiController {
|
export class PlatformApiController {
|
||||||
private logger: any;
|
private logger: any;
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { ReleaseNoteController } from './release_note.controller';
|
import { ReleaseNoteController } from './release_note.controller';
|
||||||
import { ReleaseNoteService } from './release_note.service';
|
import { ReleaseNoteService } from './release_note.service';
|
||||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
|
||||||
|
|
||||||
describe('ReleaseNoteController', () => {
|
describe('ReleaseNoteController', () => {
|
||||||
let controller: ReleaseNoteController;
|
let controller: ReleaseNoteController;
|
||||||
@@ -9,7 +8,7 @@ describe('ReleaseNoteController', () => {
|
|||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
controllers: [ReleaseNoteController],
|
controllers: [ReleaseNoteController],
|
||||||
providers: [ReleaseNoteService, DadosferaLogger],
|
providers: [ReleaseNoteService],
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
||||||
|
|||||||
@@ -1,13 +1,12 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { ReleaseNoteService } from './release_note.service';
|
import { ReleaseNoteService } from './release_note.service';
|
||||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
|
||||||
|
|
||||||
describe('ReleaseNoteService', () => {
|
describe('ReleaseNoteService', () => {
|
||||||
let service: ReleaseNoteService;
|
let service: ReleaseNoteService;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
providers: [ReleaseNoteService, DadosferaLogger],
|
providers: [ReleaseNoteService],
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
||||||
|
|||||||
+2
-5
@@ -1,8 +1,5 @@
|
|||||||
export interface Info {
|
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||||
user_id: string;
|
|
||||||
customer_id: string;
|
|
||||||
customer: string;
|
|
||||||
}
|
|
||||||
export interface ICreateTransformationsRequest {
|
export interface ICreateTransformationsRequest {
|
||||||
transformations: Transformation[];
|
transformations: Transformation[];
|
||||||
info: Info;
|
info: Info;
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ interface IMetadata {
|
|||||||
details?: string;
|
details?: string;
|
||||||
sensitive?: string;
|
sensitive?: string;
|
||||||
roles?: string[];
|
roles?: string[];
|
||||||
customer_modules?: string[];
|
|
||||||
is_data_manager?: boolean;
|
is_data_manager?: boolean;
|
||||||
access_token?: string;
|
access_token?: string;
|
||||||
host?: string;
|
host?: string;
|
||||||
|
|||||||
Reference in New Issue
Block a user