mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-08 03:14:47 +00:00
Compare commits
44
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b736cddf07 | ||
|
|
007f3911ff | ||
|
|
d3c5c0fa63 | ||
|
|
5dbc644d1d | ||
|
|
8eddb9e1bf | ||
|
|
76485f929d | ||
|
|
a18bdccc09 | ||
|
|
b27298501d | ||
|
|
33ebc91826 | ||
|
|
6948156693 | ||
|
|
0aaa4384c3 | ||
|
|
1f9d0c29ec | ||
|
|
039c652b28 | ||
|
|
653d4f53b5 | ||
|
|
31f8c2c1a6 | ||
|
|
8b93d4e97b | ||
|
|
adeb022818 | ||
|
|
e326cab44d | ||
|
|
61109f8ae9 | ||
|
|
3f8dc5cabe | ||
|
|
7f5d157739 | ||
|
|
bce73fb11f | ||
|
|
9473e65deb | ||
|
|
77b9acd2d0 | ||
|
|
e99306adba | ||
|
|
96b947ebdc | ||
|
|
6f7436f33f | ||
|
|
865140e681 | ||
|
|
c4a664572a | ||
|
|
7a10f88113 | ||
|
|
466f8fb8cc | ||
|
|
5989822263 | ||
|
|
64a3e2652e | ||
|
|
ea44a1cbb6 | ||
|
|
fc9c0b0991 | ||
|
|
a4b5a44e44 | ||
|
|
d99a6aa322 | ||
|
|
3f910f851a | ||
|
|
bd231382eb | ||
|
|
9c1979e17a | ||
|
|
fcf7fb054e | ||
|
|
288aaeabc0 | ||
|
|
918c3d7416 | ||
|
|
e7f410831f |
+2
-1
@@ -1,4 +1,5 @@
|
|||||||
FROM node:18.17-alpine AS base_image
|
FROM node:20-alpine AS base_image
|
||||||
|
RUN npm install -g npm@latest
|
||||||
|
|
||||||
FROM base_image AS build_base
|
FROM base_image AS build_base
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
FROM node:20-alpine AS base_image
|
||||||
|
RUN npm install -g npm@latest
|
||||||
|
|
||||||
|
FROM base_image AS build_base
|
||||||
|
WORKDIR /app
|
||||||
|
RUN apk update
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
aws-cli \
|
||||||
|
chromium \
|
||||||
|
nss \
|
||||||
|
freetype \
|
||||||
|
harfbuzz \
|
||||||
|
ca-certificates \
|
||||||
|
ttf-freefont
|
||||||
|
COPY package*.json ./
|
||||||
|
|
||||||
|
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||||
|
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||||
|
|
||||||
|
|
||||||
|
# Local build with secrets
|
||||||
|
FROM build_base AS build
|
||||||
|
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
||||||
|
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||||
|
RUN npm ci
|
||||||
|
COPY . .
|
||||||
|
RUN npm run build
|
||||||
|
|
||||||
|
|
||||||
|
FROM base_image
|
||||||
|
WORKDIR /app
|
||||||
|
COPY --from=build /app/dist ./dist
|
||||||
|
COPY --from=build /app/node_modules ./node_modules
|
||||||
|
COPY --from=build /app/package*.json ./
|
||||||
|
RUN apk update
|
||||||
|
RUN apk add --no-cache \
|
||||||
|
chromium \
|
||||||
|
nss \
|
||||||
|
freetype \
|
||||||
|
harfbuzz \
|
||||||
|
ca-certificates \
|
||||||
|
ttf-freefont
|
||||||
|
|
||||||
|
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||||
|
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||||
|
|
||||||
|
ENTRYPOINT ["npm", "run", "start:prod"]
|
||||||
+62
-30
@@ -1321,9 +1321,6 @@
|
|||||||
"Users"
|
"Users"
|
||||||
],
|
],
|
||||||
"security": [
|
"security": [
|
||||||
{
|
|
||||||
"access-token": []
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"access-token": []
|
"access-token": []
|
||||||
}
|
}
|
||||||
@@ -4483,6 +4480,14 @@
|
|||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "asset_type",
|
||||||
|
"required": true,
|
||||||
|
"in": "query",
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"responses": {
|
"responses": {
|
||||||
@@ -7217,33 +7222,6 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"/platform/pipeline/pipeline_run": {
|
|
||||||
"post": {
|
|
||||||
"operationId": "PlatformApiController_createPipelineRun",
|
|
||||||
"summary": "Create a pipeline run",
|
|
||||||
"parameters": [],
|
|
||||||
"responses": {
|
|
||||||
"201": {
|
|
||||||
"description": "",
|
|
||||||
"content": {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"type": "object"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"tags": [
|
|
||||||
"Platform API"
|
|
||||||
],
|
|
||||||
"security": [
|
|
||||||
{
|
|
||||||
"access-token": []
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"/platform/pipeline/{pipelineId}/pipeline_run": {
|
"/platform/pipeline/{pipelineId}/pipeline_run": {
|
||||||
"get": {
|
"get": {
|
||||||
"operationId": "PlatformApiController_getPipelineRuns",
|
"operationId": "PlatformApiController_getPipelineRuns",
|
||||||
@@ -8185,6 +8163,15 @@
|
|||||||
"hierarchy": {
|
"hierarchy": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"bio": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companyName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companySite": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"mfaStatus": {
|
"mfaStatus": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
@@ -8215,6 +8202,9 @@
|
|||||||
"jobTitle",
|
"jobTitle",
|
||||||
"department",
|
"department",
|
||||||
"hierarchy",
|
"hierarchy",
|
||||||
|
"bio",
|
||||||
|
"companyName",
|
||||||
|
"companySite",
|
||||||
"mfaStatus",
|
"mfaStatus",
|
||||||
"createdAt",
|
"createdAt",
|
||||||
"updatedAt",
|
"updatedAt",
|
||||||
@@ -8348,6 +8338,15 @@
|
|||||||
"hierarchy": {
|
"hierarchy": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"bio": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companyName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companySite": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"customer": {
|
"customer": {
|
||||||
"$ref": "#/components/schemas/Customer"
|
"$ref": "#/components/schemas/Customer"
|
||||||
},
|
},
|
||||||
@@ -8375,6 +8374,9 @@
|
|||||||
"jobTitle",
|
"jobTitle",
|
||||||
"department",
|
"department",
|
||||||
"hierarchy",
|
"hierarchy",
|
||||||
|
"bio",
|
||||||
|
"companyName",
|
||||||
|
"companySite",
|
||||||
"mfaStatus",
|
"mfaStatus",
|
||||||
"createdAt",
|
"createdAt",
|
||||||
"updatedAt",
|
"updatedAt",
|
||||||
@@ -8515,6 +8517,15 @@
|
|||||||
"hierarchy": {
|
"hierarchy": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"bio": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companySite": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companyName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"roleNames": {
|
"roleNames": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
@@ -8555,6 +8566,15 @@
|
|||||||
"hierarchy": {
|
"hierarchy": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"bio": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companyName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"companySite": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"mfaStatus": {
|
"mfaStatus": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
@@ -8578,6 +8598,9 @@
|
|||||||
"jobTitle",
|
"jobTitle",
|
||||||
"department",
|
"department",
|
||||||
"hierarchy",
|
"hierarchy",
|
||||||
|
"bio",
|
||||||
|
"companyName",
|
||||||
|
"companySite",
|
||||||
"mfaStatus",
|
"mfaStatus",
|
||||||
"createdAt",
|
"createdAt",
|
||||||
"updatedAt",
|
"updatedAt",
|
||||||
@@ -9626,6 +9649,9 @@
|
|||||||
"public",
|
"public",
|
||||||
"private"
|
"private"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"docs": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
@@ -9880,6 +9906,9 @@
|
|||||||
"public",
|
"public",
|
||||||
"private"
|
"private"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"docs": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
@@ -9919,6 +9948,9 @@
|
|||||||
},
|
},
|
||||||
"embed": {
|
"embed": {
|
||||||
"$ref": "#/components/schemas/EmbedObject"
|
"$ref": "#/components/schemas/EmbedObject"
|
||||||
|
},
|
||||||
|
"docs": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
|
|||||||
Generated
+2110
-2969
File diff suppressed because it is too large
Load Diff
+10
-4
@@ -30,12 +30,12 @@
|
|||||||
"@aws-crypto/sha256-js": "^5.2.0",
|
"@aws-crypto/sha256-js": "^5.2.0",
|
||||||
"@aws-sdk/client-dynamodb": "^3.414.0",
|
"@aws-sdk/client-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/client-secrets-manager": "^3.414.0",
|
"@aws-sdk/client-secrets-manager": "^3.414.0",
|
||||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
|
||||||
"@aws-sdk/credential-provider-node": "^3.940.0",
|
"@aws-sdk/credential-provider-node": "^3.940.0",
|
||||||
|
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack": "2.5.3",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
"@dadosfera/protospack-v2": "3.38.0-beta.16",
|
"@dadosfera/protospack-v2": "3.38.0-beta.19",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
@@ -49,7 +49,7 @@
|
|||||||
"@nestjs/schematics": "^9.2.0",
|
"@nestjs/schematics": "^9.2.0",
|
||||||
"@nestjs/swagger": "^6.3.0",
|
"@nestjs/swagger": "^6.3.0",
|
||||||
"@nestjs/testing": "^9.4.3",
|
"@nestjs/testing": "^9.4.3",
|
||||||
"axios": "^0.27.2",
|
"axios": "^0.30.2",
|
||||||
"cache-manager": "^5.1.4",
|
"cache-manager": "^5.1.4",
|
||||||
"cache-manager-ioredis-yet": "^1.1.0",
|
"cache-manager-ioredis-yet": "^1.1.0",
|
||||||
"class-transformer": "^0.5.1",
|
"class-transformer": "^0.5.1",
|
||||||
@@ -80,7 +80,13 @@
|
|||||||
"swagger-ui-express": "^4.6.3"
|
"swagger-ui-express": "^4.6.3"
|
||||||
},
|
},
|
||||||
"overrides": {
|
"overrides": {
|
||||||
"multer": "1.4.5-lts.1"
|
"multer": "2.0.2",
|
||||||
|
"form-data": "^4.0.4",
|
||||||
|
"body-parser": "^1.20.3",
|
||||||
|
"cross-spawn": "^7.0.5",
|
||||||
|
"glob": "^10.5.0",
|
||||||
|
"path-to-regexp": "^3.3.0",
|
||||||
|
"semver": "^7.5.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/cache-manager": "^4.0.6",
|
"@types/cache-manager": "^4.0.6",
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ import {
|
|||||||
import { TypeParser } from 'src/utils/FileParser/parser-types';
|
import { TypeParser } from 'src/utils/FileParser/parser-types';
|
||||||
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
|
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
|
||||||
|
|
||||||
|
|
||||||
class CatalogService implements OnModuleInit {
|
class CatalogService implements OnModuleInit {
|
||||||
catalogReadService: ReadService.CatalogReadServices;
|
catalogReadService: ReadService.CatalogReadServices;
|
||||||
catalogWriteService: WriteService.CatalogWriteServices;
|
catalogWriteService: WriteService.CatalogWriteServices;
|
||||||
@@ -58,7 +57,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
this.logger = dadosferaLogger.logger;
|
this.logger = dadosferaLogger.logger;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
onModuleInit() {
|
onModuleInit() {
|
||||||
this.catalogReadService =
|
this.catalogReadService =
|
||||||
this.grpcClient.getService<ReadService.CatalogReadServices>(
|
this.grpcClient.getService<ReadService.CatalogReadServices>(
|
||||||
@@ -74,81 +72,65 @@ class CatalogService implements OnModuleInit {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
_getNimbusUrl(body) {
|
_getNimbusUrl(body) {
|
||||||
this.logger.debug(`Body: ${JSON.stringify(body)}`);
|
this.logger.debug(`Body: ${JSON.stringify(body)}`);
|
||||||
const customer = body.info.customer.toLowerCase();
|
const customer = body.info.customer.toLowerCase();
|
||||||
|
|
||||||
|
|
||||||
if (process.env.ENV === 'prd') {
|
if (process.env.ENV === 'prd') {
|
||||||
return `https://nimbus-${customer}.dadosfera.ai`;
|
return `https://nimbus-${customer}.dadosfera.ai`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return `https://nimbus-${customer}.${process.env.ENV.replace(
|
return `https://nimbus-${customer}.${process.env.ENV.replace(
|
||||||
'local',
|
'local',
|
||||||
'stg',
|
'stg',
|
||||||
)}.dadosfera.ai`;
|
)}.dadosfera.ai`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getPiiReporter(metadata: Metadata, type: TypeParser) {
|
async getPiiReporter(metadata: Metadata, type: TypeParser) {
|
||||||
this.logger.info('getPiiReporter: ' + type)
|
this.logger.info('getPiiReporter: ' + type);
|
||||||
try {
|
try {
|
||||||
const {
|
const { data } = await lastValueFrom(
|
||||||
data
|
this.catalogWriteService.GetPiiReporter({}, metadata),
|
||||||
} = await lastValueFrom(
|
);
|
||||||
this.catalogWriteService.GetPiiReporter({}, metadata)
|
this.logger.info('Finish grpc call');
|
||||||
)
|
|
||||||
this.logger.info("Finish grpc call")
|
|
||||||
|
|
||||||
|
|
||||||
const parser = ParserBuilder.build<PiiMetadata>(type);
|
const parser = ParserBuilder.build<PiiMetadata>(type);
|
||||||
|
|
||||||
|
this.logger.info('parser file to: ' + type);
|
||||||
this.logger.info('parser file to: ' + type)
|
const file = await parser.parse(data);
|
||||||
const file = await parser.parse(data)
|
this.logger.info('finish parser');
|
||||||
this.logger.info('finish parser')
|
|
||||||
const mimeTypes: Record<TypeParser, string> = {
|
const mimeTypes: Record<TypeParser, string> = {
|
||||||
'csv': 'text/csv',
|
csv: 'text/csv',
|
||||||
'html': 'text/html',
|
html: 'text/html',
|
||||||
'pdf': 'application/pdf'
|
pdf: 'application/pdf',
|
||||||
}
|
};
|
||||||
|
|
||||||
|
|
||||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||||
const filename = `relatorio-pii-${timestamp}.${type}`;
|
const filename = `relatorio-pii-${timestamp}.${type}`;
|
||||||
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
file,
|
file,
|
||||||
filename: filename,
|
filename: filename,
|
||||||
type: mimeTypes[type]
|
type: mimeTypes[type],
|
||||||
}
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(error.message);
|
this.logger.error(error.message);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
|
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
|
||||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||||
if (!data.embed) data.embed = undefined;
|
if (!data.embed) data.embed = undefined;
|
||||||
|
|
||||||
|
|
||||||
return lastValueFrom(
|
return lastValueFrom(
|
||||||
this.catalogWriteService.CreateDataAsset(data, metadata),
|
this.catalogWriteService.CreateDataAsset(data, metadata),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
|
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
|
||||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||||
|
|
||||||
|
|
||||||
return lastValueFrom(
|
return lastValueFrom(
|
||||||
this.catalogWriteService.ManagePermission(data, metadata),
|
this.catalogWriteService.ManagePermission(data, metadata),
|
||||||
).catch((err) => {
|
).catch((err) => {
|
||||||
@@ -159,11 +141,9 @@ class CatalogService implements OnModuleInit {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
|
async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
|
||||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||||
|
|
||||||
|
|
||||||
return lastValueFrom(
|
return lastValueFrom(
|
||||||
this.catalogWriteService.RevokePermission(data, metadata),
|
this.catalogWriteService.RevokePermission(data, metadata),
|
||||||
).catch((err) => {
|
).catch((err) => {
|
||||||
@@ -174,11 +154,9 @@ class CatalogService implements OnModuleInit {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
|
async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
|
||||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||||
|
|
||||||
|
|
||||||
return lastValueFrom(
|
return lastValueFrom(
|
||||||
this.catalogWriteService.MakeAComment(data, metadata),
|
this.catalogWriteService.MakeAComment(data, metadata),
|
||||||
).catch((err) => {
|
).catch((err) => {
|
||||||
@@ -189,11 +167,9 @@ class CatalogService implements OnModuleInit {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
|
async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
|
||||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||||
|
|
||||||
|
|
||||||
return lastValueFrom(
|
return lastValueFrom(
|
||||||
this.catalogWriteService.UpdateAComment(data, metadata),
|
this.catalogWriteService.UpdateAComment(data, metadata),
|
||||||
).catch((err) => {
|
).catch((err) => {
|
||||||
@@ -204,11 +180,9 @@ class CatalogService implements OnModuleInit {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
|
async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
|
||||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||||
|
|
||||||
|
|
||||||
return lastValueFrom(
|
return lastValueFrom(
|
||||||
this.catalogWriteService.DeleteDataAsset(data, metadata),
|
this.catalogWriteService.DeleteDataAsset(data, metadata),
|
||||||
).catch((err) => {
|
).catch((err) => {
|
||||||
@@ -219,30 +193,114 @@ class CatalogService implements OnModuleInit {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getUserRolesIds(userId: string) {
|
async getUserRolesIds(userId: string) {
|
||||||
const result = await this.userService.findOneById(userId).catch(() => null);
|
const result = await this.userService.findOneById(userId).catch(() => null);
|
||||||
|
|
||||||
|
|
||||||
if (result) {
|
if (result) {
|
||||||
return result.user.roles.map((role) => role.id);
|
return result.user.roles.map((role) => role.id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async searchDataAssets(
|
async searchDataAssets(
|
||||||
query: Record<string, any>,
|
query: Record<string, any>,
|
||||||
metadata: Metadata,
|
metadata: Metadata,
|
||||||
customer_id: string,
|
customer_id: string,
|
||||||
) {
|
) {
|
||||||
this.logger.info('CatalogService - searchDataAssets');
|
this.logger.info('CatalogService - searchDataAssets', { query });
|
||||||
|
|
||||||
|
|
||||||
const { search, page, size, sort_by, order, ...filters } = query;
|
const { search, page, size, sort_by, order, ...filters } = query;
|
||||||
|
|
||||||
|
this.logger.debug('Extracted filters:', { filters });
|
||||||
|
|
||||||
|
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
|
||||||
|
search,
|
||||||
|
page,
|
||||||
|
size,
|
||||||
|
sort_by,
|
||||||
|
order,
|
||||||
|
filters,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (
|
||||||
|
filters.manually !== undefined &&
|
||||||
|
filters.manually !== null &&
|
||||||
|
filters.manually !== ''
|
||||||
|
) {
|
||||||
|
filters.manually = Number(filters.manually); // 1 ou 0
|
||||||
|
} else {
|
||||||
|
delete filters.manually;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
|
||||||
|
search,
|
||||||
|
page,
|
||||||
|
size,
|
||||||
|
sort_by,
|
||||||
|
order,
|
||||||
|
filters,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (filters.owner) {
|
||||||
|
const { users: customer_users } =
|
||||||
|
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||||
|
|
||||||
|
this.logger.info('Available users in database count:', {
|
||||||
|
count: customer_users.length,
|
||||||
|
});
|
||||||
|
this.logger.info('First 5 users:', {
|
||||||
|
users: customer_users
|
||||||
|
.slice(0, 5)
|
||||||
|
.map((u) => ({ id: u.id, email: u.email, name: u.name })),
|
||||||
|
});
|
||||||
|
|
||||||
|
const ownerValues = Array.isArray(filters.owner)
|
||||||
|
? filters.owner
|
||||||
|
: typeof filters.owner === 'string' && filters.owner.includes(',')
|
||||||
|
? filters.owner.split(',').map((o: string) => o.trim())
|
||||||
|
: [filters.owner];
|
||||||
|
|
||||||
|
this.logger.info('Owner values to convert:', {
|
||||||
|
ownerValues,
|
||||||
|
ownerFiltersOriginal: filters.owner,
|
||||||
|
});
|
||||||
|
|
||||||
|
const ownerIds = ownerValues
|
||||||
|
.map((ownerValue: string) => {
|
||||||
|
const normalizedOwner = ownerValue.replace(/\s/g, '+');
|
||||||
|
const user = customer_users.find((u) => {
|
||||||
|
const isIdMatch = u.id === ownerValue;
|
||||||
|
const isEmailMatch =
|
||||||
|
u.email === ownerValue || u.email === normalizedOwner;
|
||||||
|
const isNameMatch =
|
||||||
|
u.name === ownerValue || u.name === normalizedOwner;
|
||||||
|
this.logger.info('Comparing:', {
|
||||||
|
userId: u.id,
|
||||||
|
userEmail: u.email,
|
||||||
|
userName: u.name,
|
||||||
|
filterValue: ownerValue,
|
||||||
|
normalizedFilter: normalizedOwner,
|
||||||
|
idMatch: isIdMatch,
|
||||||
|
emailMatch: isEmailMatch,
|
||||||
|
nameMatch: isNameMatch,
|
||||||
|
});
|
||||||
|
return isIdMatch || isEmailMatch || isNameMatch;
|
||||||
|
});
|
||||||
|
this.logger.info('Looking for owner result:', {
|
||||||
|
ownerValue,
|
||||||
|
found: !!user,
|
||||||
|
userId: user?.id,
|
||||||
|
});
|
||||||
|
return user?.id || ownerValue;
|
||||||
|
})
|
||||||
|
.filter((id: string) => id);
|
||||||
|
|
||||||
|
if (ownerIds.length > 0) {
|
||||||
|
filters.owner = ownerIds;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const { data_assets, total } = await lastValueFrom(
|
const { data_assets, total } = await lastValueFrom(
|
||||||
this.catalogReadService.GetAllDataAssets(
|
this.catalogReadService.GetAllDataAssets(
|
||||||
{
|
{
|
||||||
@@ -257,20 +315,18 @@ class CatalogService implements OnModuleInit {
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
|
||||||
|
|
||||||
const result = JSON.parse(data_assets);
|
const result = JSON.parse(data_assets);
|
||||||
|
|
||||||
|
|
||||||
const response = await this.getAssetsUsersAndRoles(
|
const response = await this.getAssetsUsersAndRoles(
|
||||||
result.data_assets,
|
result.data_assets,
|
||||||
customer_id,
|
customer_id,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
||||||
return { data_assets: response, total };
|
return { data_assets: response, total };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async downloadAssets(
|
async downloadAssets(
|
||||||
query: Record<string, any>,
|
query: Record<string, any>,
|
||||||
metadata: Metadata,
|
metadata: Metadata,
|
||||||
@@ -278,33 +334,27 @@ class CatalogService implements OnModuleInit {
|
|||||||
) {
|
) {
|
||||||
const data = await this.searchDataAssets(query, metadata, customer_id);
|
const data = await this.searchDataAssets(query, metadata, customer_id);
|
||||||
|
|
||||||
|
const formatData = data.data_assets.map((asset) => ({
|
||||||
const formatData = data.data_assets.map(asset => ({
|
|
||||||
id: asset.id,
|
id: asset.id,
|
||||||
display_name: asset.display_name,
|
display_name: asset.display_name,
|
||||||
data_asset_type: asset.data_asset_type,
|
data_asset_type: asset.data_asset_type,
|
||||||
created_at: asset.created_at,
|
created_at: asset.created_at,
|
||||||
tags: '[' + asset.tags.join(', ') + ']'
|
tags: '[' + asset.tags.join(', ') + ']',
|
||||||
}))
|
}));
|
||||||
|
|
||||||
|
|
||||||
const parser = ParserBuilder.build<AssetReporter>('csv');
|
const parser = ParserBuilder.build<AssetReporter>('csv');
|
||||||
|
|
||||||
|
|
||||||
const file = await parser.parse(formatData);
|
const file = await parser.parse(formatData);
|
||||||
|
|
||||||
|
|
||||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||||
const filename = `dadosfera_assets_${timestamp}.csv`;
|
const filename = `dadosfera_assets_${timestamp}.csv`;
|
||||||
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
file,
|
file,
|
||||||
filename
|
filename,
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getOneDataAsset(data: {
|
async getOneDataAsset(data: {
|
||||||
id: string;
|
id: string;
|
||||||
customer_id: string;
|
customer_id: string;
|
||||||
@@ -325,11 +375,9 @@ class CatalogService implements OnModuleInit {
|
|||||||
};
|
};
|
||||||
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
||||||
|
|
||||||
|
|
||||||
return { data_asset: asset[0] };
|
return { data_asset: asset[0] };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getOneDataAssetByPipelineAndObject(data: {
|
async getOneDataAssetByPipelineAndObject(data: {
|
||||||
customer_id: string;
|
customer_id: string;
|
||||||
pipeline: string;
|
pipeline: string;
|
||||||
@@ -351,11 +399,9 @@ class CatalogService implements OnModuleInit {
|
|||||||
};
|
};
|
||||||
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
||||||
|
|
||||||
|
|
||||||
return { data_asset: asset[0] };
|
return { data_asset: asset[0] };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async updateOneDataAsset(data: {
|
async updateOneDataAsset(data: {
|
||||||
data_asset_id: string;
|
data_asset_id: string;
|
||||||
customer_id: string;
|
customer_id: string;
|
||||||
@@ -364,7 +410,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
}) {
|
}) {
|
||||||
const { body, customer_id, data_asset_id, metadata } = data;
|
const { body, customer_id, data_asset_id, metadata } = data;
|
||||||
|
|
||||||
|
|
||||||
const { data_asset } = await lastValueFrom(
|
const { data_asset } = await lastValueFrom(
|
||||||
this.catalogWriteService.UpdateDataAsset(
|
this.catalogWriteService.UpdateDataAsset(
|
||||||
{ id: data_asset_id, changes: JSON.stringify(body) },
|
{ id: data_asset_id, changes: JSON.stringify(body) },
|
||||||
@@ -379,7 +424,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
};
|
};
|
||||||
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
||||||
|
|
||||||
|
|
||||||
return { data_asset: asset[0] };
|
return { data_asset: asset[0] };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -387,12 +431,11 @@ class CatalogService implements OnModuleInit {
|
|||||||
const { documentation } = await lastValueFrom(
|
const { documentation } = await lastValueFrom(
|
||||||
this.catalogReadService.GetDatasetDoc({ id }, metadata),
|
this.catalogReadService.GetDatasetDoc({ id }, metadata),
|
||||||
);
|
);
|
||||||
|
|
||||||
const docs = JSON.parse(documentation);
|
const docs = JSON.parse(documentation);
|
||||||
return docs;
|
return docs;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getDatasetPreview(id: string, metadata: Metadata) {
|
async getDatasetPreview(id: string, metadata: Metadata) {
|
||||||
const { preview } = await lastValueFrom(
|
const { preview } = await lastValueFrom(
|
||||||
this.catalogReadService.GetDatasetPreview(
|
this.catalogReadService.GetDatasetPreview(
|
||||||
@@ -404,7 +447,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
|
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
|
||||||
const { columns_metadata } = await lastValueFrom(
|
const { columns_metadata } = await lastValueFrom(
|
||||||
this.catalogReadService.GetDatasetColumnsMetadata(
|
this.catalogReadService.GetDatasetColumnsMetadata(
|
||||||
@@ -420,7 +462,7 @@ class CatalogService implements OnModuleInit {
|
|||||||
if (body.asset_type === 'table' || body.asset_type === 'view') {
|
if (body.asset_type === 'table' || body.asset_type === 'view') {
|
||||||
return this.createDataDocsViaNimbus(body);
|
return this.createDataDocsViaNimbus(body);
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.createDataDocsViaGrpc(body, metadata);
|
return this.createDataDocsViaGrpc(body, metadata);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -434,7 +476,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
|
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
|
||||||
this.logger.info('Creating data docs via gRPC for other asset types');
|
this.logger.info('Creating data docs via gRPC for other asset types');
|
||||||
try {
|
try {
|
||||||
@@ -447,9 +488,8 @@ class CatalogService implements OnModuleInit {
|
|||||||
metadata,
|
metadata,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error('Error creating data asset docs:', error);
|
this.logger.error('Error creating data asset docs:', error);
|
||||||
throw new HttpException(
|
throw new HttpException(
|
||||||
@@ -462,7 +502,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
async findAllTags(data, metadata) {
|
async findAllTags(data, metadata) {
|
||||||
this.logger.info('CatalogService - findAllCustomerTags');
|
this.logger.info('CatalogService - findAllCustomerTags');
|
||||||
|
|
||||||
|
|
||||||
const response = await lastValueFrom(
|
const response = await lastValueFrom(
|
||||||
this.catalogReadService.GetCustomerTags(data, metadata),
|
this.catalogReadService.GetCustomerTags(data, metadata),
|
||||||
)
|
)
|
||||||
@@ -475,9 +514,9 @@ class CatalogService implements OnModuleInit {
|
|||||||
throw new Error(err);
|
throw new Error(err);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
|
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
|
||||||
const { users: customer_users } =
|
const { users: customer_users } =
|
||||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||||
@@ -490,17 +529,15 @@ class CatalogService implements OnModuleInit {
|
|||||||
(u) => u.id === data_asset.owner,
|
(u) => u.id === data_asset.owner,
|
||||||
)?.email;
|
)?.email;
|
||||||
|
|
||||||
|
|
||||||
const roles = [];
|
const roles = [];
|
||||||
const users = [];
|
const users = [];
|
||||||
const data_asset_roles = data_asset?.roles || []
|
const data_asset_roles = data_asset?.roles || [];
|
||||||
for (const role_id of data_asset_roles) {
|
for (const role_id of data_asset_roles) {
|
||||||
const role = customer_roles.find((r) => r.id === role_id);
|
const role = customer_roles.find((r) => r.id === role_id);
|
||||||
if (role) roles.push({ id: role.id, name: role.name });
|
if (role) roles.push({ id: role.id, name: role.name });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const data_asset_users = data_asset?.users || [];
|
||||||
const data_asset_users = data_asset?.users || []
|
|
||||||
for (const user_id of data_asset_users) {
|
for (const user_id of data_asset_users) {
|
||||||
const user = customer_users.find((r) => r.id === user_id);
|
const user = customer_users.find((r) => r.id === user_id);
|
||||||
if (user) users.push({ id: user.id, email: user.email });
|
if (user) users.push({ id: user.id, email: user.email });
|
||||||
@@ -514,7 +551,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
|
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
|
||||||
const { session } = await lastValueFrom(
|
const { session } = await lastValueFrom(
|
||||||
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
|
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
|
||||||
@@ -528,7 +564,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
|
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
|
||||||
const res = await lastValueFrom(
|
const res = await lastValueFrom(
|
||||||
this.catalogWriteService.AddRlsRule(data, metadata),
|
this.catalogWriteService.AddRlsRule(data, metadata),
|
||||||
@@ -536,7 +571,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async removeRlsRule(id: number, metadata: Metadata) {
|
async removeRlsRule(id: number, metadata: Metadata) {
|
||||||
const res = await lastValueFrom(
|
const res = await lastValueFrom(
|
||||||
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
|
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
|
||||||
@@ -544,14 +578,12 @@ class CatalogService implements OnModuleInit {
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async batchRemoveRlsRule(
|
async batchRemoveRlsRule(
|
||||||
query: BatchRemoveRlsRulesRequest,
|
query: BatchRemoveRlsRulesRequest,
|
||||||
metadata: Metadata,
|
metadata: Metadata,
|
||||||
) {
|
) {
|
||||||
const { id_rls, nimbus_dashboard_id } = query;
|
const { id_rls, nimbus_dashboard_id } = query;
|
||||||
|
|
||||||
|
|
||||||
if (id_rls && nimbus_dashboard_id) {
|
if (id_rls && nimbus_dashboard_id) {
|
||||||
throw new BadRequestException(
|
throw new BadRequestException(
|
||||||
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
|
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
|
||||||
@@ -572,7 +604,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return 'OK';
|
return 'OK';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
|
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
|
||||||
const res = await lastValueFrom(
|
const res = await lastValueFrom(
|
||||||
this.catalogReadService.GetRlsRules(data, metadata),
|
this.catalogReadService.GetRlsRules(data, metadata),
|
||||||
@@ -580,7 +611,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return res.rls_rules;
|
return res.rls_rules;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getOneRlsRule(id: number, metadata: Metadata) {
|
async getOneRlsRule(id: number, metadata: Metadata) {
|
||||||
const res = await lastValueFrom(
|
const res = await lastValueFrom(
|
||||||
this.catalogReadService.GetOneRlsRule({ id }, metadata),
|
this.catalogReadService.GetOneRlsRule({ id }, metadata),
|
||||||
@@ -588,7 +618,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return res.rls_rule;
|
return res.rls_rule;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async getNimbusDashboards(
|
async getNimbusDashboards(
|
||||||
data: GetNimbusDashboardsRequest,
|
data: GetNimbusDashboardsRequest,
|
||||||
metadata: Metadata,
|
metadata: Metadata,
|
||||||
@@ -599,88 +628,99 @@ class CatalogService implements OnModuleInit {
|
|||||||
return res.dashboards;
|
return res.dashboards;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async createTableMetadata(body: any): Promise<number> {
|
async createTableMetadata(body: any): Promise<number> {
|
||||||
const nimbusUrl = this._getNimbusUrl(body);
|
const nimbusUrl = this._getNimbusUrl(body);
|
||||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
|
||||||
|
|
||||||
|
|
||||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
|
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
|
`Creating table metadata for table ${body.table_metadata.table_name}`,
|
||||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
{ ...body.logMetadata },
|
||||||
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
|
);
|
||||||
|
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
|
||||||
|
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {
|
||||||
|
...body.logMetadata,
|
||||||
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
|
const { data, status } = await axios.post(endpoint, {
|
||||||
|
...body.table_metadata,
|
||||||
|
});
|
||||||
|
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
|
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
|
||||||
{...body.logMetadata},
|
{ ...body.logMetadata },
|
||||||
);
|
);
|
||||||
return data.id;
|
return data.id;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
|
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
|
||||||
error.response?.status
|
error.response?.status
|
||||||
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
|
} because of ${JSON.stringify(error.response?.data) || error.message}`,
|
||||||
|
{ ...body.logMetadata },
|
||||||
|
);
|
||||||
throw new Error(error.response?.data?.message || error.message);
|
throw new Error(error.response?.data?.message || error.message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async createColumnMetadata(body: any): Promise<number[]> {
|
async createColumnMetadata(body: any): Promise<number[]> {
|
||||||
const nimbusUrl = this._getNimbusUrl(body);
|
const nimbusUrl = this._getNimbusUrl(body);
|
||||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
|
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
|
||||||
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
|
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
|
this.logger.info(
|
||||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
`Creating column metadata for table ${body.column_metadata.table_name}`,
|
||||||
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
|
{ ...body.logMetadata },
|
||||||
const { data, status } = await axios.post(endpoint, body.column_metadata);
|
);
|
||||||
|
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
|
||||||
|
this.logger.debug(
|
||||||
|
`Payload: ${JSON.stringify(body.column_metadata)}`,
|
||||||
|
{ ...body.logMetadata },
|
||||||
|
);
|
||||||
|
const { data, status } = await axios.post(
|
||||||
|
endpoint,
|
||||||
|
body.column_metadata,
|
||||||
|
);
|
||||||
|
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
|
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
|
||||||
{...body.logMetadata},
|
{ ...body.logMetadata },
|
||||||
);
|
);
|
||||||
return data.map((column) => column.id);
|
return data.map((column) => column.id);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error(
|
this.logger.error(
|
||||||
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
|
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
|
||||||
error.response?.status
|
error.response?.status
|
||||||
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
|
} because of ${error.response?.data || error.message}`,
|
||||||
|
{ ...body.logMetadata },
|
||||||
|
);
|
||||||
throw new Error(error.response?.data?.message || error.message);
|
throw new Error(error.response?.data?.message || error.message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async createDataPreview(body: any): Promise<number> {
|
async createDataPreview(body: any): Promise<number> {
|
||||||
const nimbusUrl = this._getNimbusUrl(body);
|
const nimbusUrl = this._getNimbusUrl(body);
|
||||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
|
||||||
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
|
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
|
`Creating data preview for table ${body.data_preview.table_name}`,
|
||||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
{ ...body.logMetadata },
|
||||||
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
|
);
|
||||||
|
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
|
||||||
|
this.logger.debug(
|
||||||
|
`Payload: ${JSON.stringify(body.data_preview)}`,
|
||||||
|
{ ...body.logMetadata },
|
||||||
|
);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { data, status } = await axios.post(endpoint, body.data_preview);
|
const { data, status } = await axios.post(endpoint, body.data_preview);
|
||||||
|
|
||||||
|
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
|
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
|
||||||
{...body.logMetadata},
|
{ ...body.logMetadata },
|
||||||
);
|
);
|
||||||
return data.id;
|
return data.id;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -688,17 +728,15 @@ class CatalogService implements OnModuleInit {
|
|||||||
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
|
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
|
||||||
error.response?.status
|
error.response?.status
|
||||||
} because of ${error.response?.data || error.message}`,
|
} because of ${error.response?.data || error.message}`,
|
||||||
{...body.logMetadata},
|
{ ...body.logMetadata },
|
||||||
);
|
);
|
||||||
throw new Error(error.response?.data?.message || error.message);
|
throw new Error(error.response?.data?.message || error.message);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||||
const customer_name_raw = metadata.get('customer_name');
|
const customer_name_raw = metadata.get('customer_name');
|
||||||
|
|
||||||
|
|
||||||
const customer_name = customer_name_raw?.[0]?.toString();
|
const customer_name = customer_name_raw?.[0]?.toString();
|
||||||
if (!customer_name) {
|
if (!customer_name) {
|
||||||
throw new BadRequestException('Customer name not found in metadata');
|
throw new BadRequestException('Customer name not found in metadata');
|
||||||
@@ -721,5 +759,4 @@ class CatalogService implements OnModuleInit {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export { CatalogService };
|
export { CatalogService };
|
||||||
|
|||||||
@@ -98,6 +98,8 @@ export class IDataAsset {
|
|||||||
embed?: EmbedObject;
|
embed?: EmbedObject;
|
||||||
@ApiPropertyOptional({ enum: DataAssetShareType })
|
@ApiPropertyOptional({ enum: DataAssetShareType })
|
||||||
share_type?: DataAssetShareType;
|
share_type?: DataAssetShareType;
|
||||||
|
@ApiPropertyOptional()
|
||||||
|
docs?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class IOneDataAsset {
|
export class IOneDataAsset {
|
||||||
@@ -200,6 +202,8 @@ export class IUpdateDataRequest {
|
|||||||
embed: EmbedObject;
|
embed: EmbedObject;
|
||||||
@ApiPropertyOptional({ enum: DataAssetShareType })
|
@ApiPropertyOptional({ enum: DataAssetShareType })
|
||||||
share_type?: DataAssetShareType;
|
share_type?: DataAssetShareType;
|
||||||
|
@ApiPropertyOptional()
|
||||||
|
docs?: string;
|
||||||
}
|
}
|
||||||
export class ICreateDataAsset implements CreateDataAssetRequest {
|
export class ICreateDataAsset implements CreateDataAssetRequest {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
@@ -214,6 +218,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
|
|||||||
location: string;
|
location: string;
|
||||||
@ApiPropertyOptional()
|
@ApiPropertyOptional()
|
||||||
embed: EmbedObject;
|
embed: EmbedObject;
|
||||||
|
@ApiPropertyOptional()
|
||||||
|
docs: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class IPreview {
|
export class IPreview {
|
||||||
|
|||||||
@@ -229,27 +229,33 @@ export class RolesService {
|
|||||||
const [roleTreated] = this.getRolesPermissionsName([role.role]);
|
const [roleTreated] = this.getRolesPermissionsName([role.role]);
|
||||||
return { role: roleTreated };
|
return { role: roleTreated };
|
||||||
}
|
}
|
||||||
|
|
||||||
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
|
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
|
||||||
const newRoles: RoleDto[] = [];
|
const newRoles: RoleDto[] = [];
|
||||||
for (const role of roles) {
|
for (const role of roles) {
|
||||||
const allPermissions = this.permissionsService.getAllPermissions(
|
const newRole: RoleDto = this.formatRole(role);
|
||||||
this.language,
|
|
||||||
);
|
|
||||||
const newPermissions = role.permissions.map((p) => {
|
|
||||||
const permission = allPermissions.find((per) => per.seqid === p.seqid);
|
|
||||||
return {
|
|
||||||
...p,
|
|
||||||
name: permission.name,
|
|
||||||
id: p.seqid,
|
|
||||||
};
|
|
||||||
});
|
|
||||||
const newRole: RoleDto = {
|
|
||||||
...role,
|
|
||||||
permissions: newPermissions,
|
|
||||||
isPublic: role.isPublic,
|
|
||||||
};
|
|
||||||
newRoles.push(newRole);
|
newRoles.push(newRole);
|
||||||
}
|
}
|
||||||
return newRoles;
|
return newRoles;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private formatRole(role: GetRolesPermissionsName): RoleDto {
|
||||||
|
const allPermissions = this.permissionsService.getAllPermissions(
|
||||||
|
this.language
|
||||||
|
);
|
||||||
|
const newPermissions = role.permissions.map((p) => {
|
||||||
|
const permission = allPermissions.find((per) => per.seqid === p.seqid);
|
||||||
|
return {
|
||||||
|
...p,
|
||||||
|
name: permission.name,
|
||||||
|
id: p.seqid,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
...role,
|
||||||
|
permissions: newPermissions,
|
||||||
|
isPublic: role.isPublic,
|
||||||
|
};;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,6 +38,12 @@ export class User {
|
|||||||
department?: string;
|
department?: string;
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
hierarchy?: string;
|
hierarchy?: string;
|
||||||
|
@ApiProperty()
|
||||||
|
bio?: string;
|
||||||
|
@ApiProperty()
|
||||||
|
companyName?: string;
|
||||||
|
@ApiProperty()
|
||||||
|
companySite?: string;
|
||||||
@ApiPropertyOptional()
|
@ApiPropertyOptional()
|
||||||
customer?: Customer;
|
customer?: Customer;
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
@@ -109,6 +115,12 @@ export class UpdateUserReq {
|
|||||||
@ApiPropertyOptional()
|
@ApiPropertyOptional()
|
||||||
hierarchy?: string;
|
hierarchy?: string;
|
||||||
@ApiPropertyOptional()
|
@ApiPropertyOptional()
|
||||||
|
bio?: string;
|
||||||
|
@ApiPropertyOptional()
|
||||||
|
companySite?: string;
|
||||||
|
@ApiPropertyOptional()
|
||||||
|
companyName?: string;
|
||||||
|
@ApiPropertyOptional()
|
||||||
roleNames?: string[];
|
roleNames?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -266,7 +266,6 @@ export class UsersController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Patch(':id')
|
@Patch(':id')
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
|
||||||
@ApiOkResponse({ type: UpdateUserRes })
|
@ApiOkResponse({ type: UpdateUserRes })
|
||||||
async updateUser(
|
async updateUser(
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
@@ -274,6 +273,18 @@ export class UsersController {
|
|||||||
@Param('id') id: string,
|
@Param('id') id: string,
|
||||||
@Language() language: LanguageEnum,
|
@Language() language: LanguageEnum,
|
||||||
) {
|
) {
|
||||||
|
|
||||||
|
const isSameUser = user.user_id === id;
|
||||||
|
const isSuperAdmin = user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)
|
||||||
|
if (!isSameUser && !isSuperAdmin) {
|
||||||
|
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isSameUser && !isSuperAdmin && body.roleNames) {
|
||||||
|
// Prevent users from updating their own roles
|
||||||
|
delete body.roleNames;
|
||||||
|
}
|
||||||
|
|
||||||
this.logger.info('updateUser', { user });
|
this.logger.info('updateUser', { user });
|
||||||
this.userService.setLanguage(language);
|
this.userService.setLanguage(language);
|
||||||
return await this.userService.updateUser(body, id, user.customer_id);
|
return await this.userService.updateUser(body, id, user.customer_id);
|
||||||
|
|||||||
@@ -148,20 +148,23 @@ export class UsersService implements OnModuleInit {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
|
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
|
||||||
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
|
const { roleNames, ...updateUserDTO } = req;
|
||||||
if (roleNames) {
|
if (roleNames && roleNames.length > 0) {
|
||||||
await this.setRoles({ roleNames, userId: id }, customerId);
|
await this.setRoles({ roleNames, userId: id }, customerId);
|
||||||
}
|
}
|
||||||
|
|
||||||
const { user } = await lastValueFrom(
|
const { user } = await lastValueFrom(
|
||||||
this.usersClientService.UserUpdate({
|
this.usersClientService.UserUpdate({
|
||||||
name,
|
department: updateUserDTO.department,
|
||||||
|
email: updateUserDTO.email,
|
||||||
|
hierarchy: updateUserDTO.hierarchy,
|
||||||
|
jobTitle: updateUserDTO.jobTitle,
|
||||||
|
name: updateUserDTO.name,
|
||||||
|
bio: updateUserDTO.bio,
|
||||||
|
companyName: updateUserDTO.companyName,
|
||||||
|
companySite: updateUserDTO.companySite,
|
||||||
customerId,
|
customerId,
|
||||||
id,
|
id,
|
||||||
department,
|
|
||||||
hierarchy,
|
|
||||||
jobTitle,
|
|
||||||
email,
|
|
||||||
metabaseUserId: undefined,
|
metabaseUserId: undefined,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ export class ValidationPipe implements PipeTransform<any> {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
const object = plainToInstance(metatype, value);
|
const object = plainToInstance(metatype, value);
|
||||||
const errors = await validate(object);
|
const errors = await validate(object, {
|
||||||
|
forbidUnknownValues: false,
|
||||||
|
whitelist: true,
|
||||||
|
});
|
||||||
if (errors.length > 0) {
|
if (errors.length > 0) {
|
||||||
const errorMessages = errors.map((err) => err.constraints);
|
const errorMessages = errors.map((err) => err.constraints);
|
||||||
throw new BadRequestException(errorMessages);
|
throw new BadRequestException(errorMessages);
|
||||||
|
|||||||
Reference in New Issue
Block a user