mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-01 04:08:16 +00:00
Compare commits
48
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f661055ae8 | ||
|
|
67a15b36b3 | ||
|
|
f396cdb2d3 | ||
|
|
01da6becff | ||
|
|
003c9b176c | ||
|
|
0a5315fb95 | ||
|
|
5a4d578d13 | ||
|
|
60214f0e57 | ||
|
|
3a2ce72910 | ||
|
|
074d3ce201 | ||
|
|
05c80e2497 | ||
|
|
8a363dd815 | ||
|
|
1e5fe59d9c | ||
|
|
1a2b9617fe | ||
|
|
1f92510a43 | ||
|
|
f7fceb1ea2 | ||
|
|
2aec60a354 | ||
|
|
9bc42a4af0 | ||
|
|
5c98cc1748 | ||
|
|
9ae14ab04a | ||
|
|
5d014f4669 | ||
|
|
5e8dea7255 | ||
|
|
38fc9f521b | ||
|
|
80d7ac6ff9 | ||
|
|
9bacd7a0f5 | ||
|
|
c89ac40039 | ||
|
|
6d6cf590ee | ||
|
|
f063a8f30e | ||
|
|
03536a2180 | ||
|
|
0efd5cc257 | ||
|
|
9439f13b7b | ||
|
|
d936806406 | ||
|
|
9d7ccd4201 | ||
|
|
c8a4c1aae6 | ||
|
|
406db7ccff | ||
|
|
f48a35c7ce | ||
|
|
8d25df74b7 | ||
|
|
831a03402e | ||
|
|
a631338599 | ||
|
|
d943721b4a | ||
|
|
1bbf978cc4 | ||
|
|
856f913ce6 | ||
|
|
dc617b6925 | ||
|
|
3f80062672 | ||
|
|
07eb423814 | ||
|
|
721232448e | ||
|
|
0b0d261aae | ||
|
|
87a8d741d7 |
+164
@@ -542,6 +542,123 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/auth/users/block": {
|
||||
"post": {
|
||||
"operationId": "AuthController_blockUsers",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Auth"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/auth/users/unblock": {
|
||||
"post": {
|
||||
"operationId": "AuthController_unblockUsers",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Auth"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/auth/users/reset": {
|
||||
"post": {
|
||||
"operationId": "AuthController_resetUsers",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Auth"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/connections": {
|
||||
"post": {
|
||||
"operationId": "ConnectionController_createConnection",
|
||||
@@ -4787,6 +4904,41 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"/catalog/register-dataset": {
|
||||
"post": {
|
||||
"operationId": "CatalogController_registerDatasetWithMetadataRequest",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dadosfera-lang",
|
||||
"in": "header",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"pt-br",
|
||||
"en-us"
|
||||
],
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"Catalog"
|
||||
],
|
||||
"security": [
|
||||
{
|
||||
"access-token": []
|
||||
},
|
||||
{
|
||||
"access-token": []
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/terms-of-use/token": {
|
||||
"get": {
|
||||
"operationId": "TermsOfUseController_getToken",
|
||||
@@ -5454,6 +5606,14 @@
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "origin",
|
||||
"required": true,
|
||||
"in": "header",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
@@ -5608,6 +5768,9 @@
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"displayName": {
|
||||
"type": "string"
|
||||
},
|
||||
"tier": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -5628,6 +5791,7 @@
|
||||
"modules",
|
||||
"id",
|
||||
"name",
|
||||
"displayName",
|
||||
"tier",
|
||||
"scheduleLimit",
|
||||
"links",
|
||||
|
||||
+4
-2
@@ -15,6 +15,8 @@ charts:
|
||||
- name: maestro.tr_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.open_customer_id
|
||||
value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
|
||||
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||
- name: maestro.open_group_id
|
||||
value: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
||||
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
|
||||
- name: replicaCount
|
||||
value: 2
|
||||
|
||||
+3
-3
@@ -9,11 +9,11 @@ charts:
|
||||
- name: hostname
|
||||
value: maestro.stg.dadosfera.ai
|
||||
- name: maestro.pi_factory_url
|
||||
value: pi-factory.dadosfera.ai
|
||||
value: pi-factory.stg.dadosfera.ai
|
||||
- name: maestro.in_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
value: in-factory.stg.dadosfera.ai
|
||||
- name: maestro.tr_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
value: in-factory.stg.dadosfera.ai
|
||||
- name: maestro.open_customer_id
|
||||
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||
- name: maestro.open_group_id
|
||||
|
||||
@@ -35,7 +35,8 @@ spec:
|
||||
- key: application
|
||||
operator: In
|
||||
values:
|
||||
- backend
|
||||
- general
|
||||
|
||||
tolerations:
|
||||
- key: "kubernetes.azure.com/scalesetpriority"
|
||||
operator: "Equal"
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
nginx.ingress.kubernetes.io/server-snippet: |
|
||||
underscores_in_headers on;
|
||||
ignore_invalid_headers on;
|
||||
|
||||
generation: 1
|
||||
labels:
|
||||
app: maestro
|
||||
name: open-data
|
||||
namespace: applications
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: {{ .Values.hostname }}
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: maestro
|
||||
port:
|
||||
number: {{ .Values.ingress.port }}
|
||||
path: /open-data/sharing-ocean-data
|
||||
pathType: Prefix
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
# Declare variables to be passed into your templates.
|
||||
|
||||
replicaCount: 3
|
||||
hostname: maestro-temp.dadosfera.ai
|
||||
hostname: maestro.dadosfera.ai
|
||||
image:
|
||||
repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
Generated
+2891
-1612
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -30,7 +30,7 @@
|
||||
"@aws-sdk/client-secrets-manager": "^3.414.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "3.37.0-beta.5",
|
||||
"@dadosfera/protospack-v2": "3.37.0-alpha.19",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
AuthRefreshAccessTokenRes,
|
||||
AuthSignInReq,
|
||||
AuthSignInRes,
|
||||
BulkEditRequest,
|
||||
} from './dtos/login';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
@@ -53,6 +54,7 @@ import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
|
||||
|
||||
|
||||
@ApiTags('Auth')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||
@@ -61,6 +63,7 @@ export class AuthController {
|
||||
logger: DadosferaLogger;
|
||||
redirectUrl: string;
|
||||
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
@@ -344,4 +347,67 @@ export class AuthController {
|
||||
|
||||
return { token, email, url, language };
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Post('users/block')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async blockUsers(
|
||||
@Language() language: LanguageEnum,
|
||||
@User() user: RequestUser,
|
||||
@Body() body: BulkEditRequest
|
||||
) {
|
||||
this.logger.info('blockUsers - Starting request');
|
||||
|
||||
try {
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
this.logger.debug('Calling blockUsers service', {
|
||||
metadata: {
|
||||
access_token: metadata.get('access_token'),
|
||||
language: metadata.get('language')
|
||||
}
|
||||
});
|
||||
|
||||
const result = await this.authClient.blockUsers(body.users, metadata);
|
||||
this.logger.info('blockUsers - Success', { result });
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error('blockUsers - Error', {
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Post('users/unblock')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async unblockUsers(
|
||||
@Language() language: LanguageEnum,
|
||||
@User() user: RequestUser,
|
||||
@Body() body: BulkEditRequest
|
||||
) {
|
||||
this.logger.info('unblockUsers');
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
return this.authClient.unblockUsers(body.users, metadata);
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Post('users/reset')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async resetUsers(
|
||||
@Language() language: LanguageEnum,
|
||||
@User() user: RequestUser,
|
||||
@Body() body: BulkEditRequest
|
||||
) {
|
||||
this.logger.info('resetUsers');
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
return this.authClient.resetUsers(body.users, metadata);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { BulkEditResponse } from './dtos/login';
|
||||
|
||||
@Injectable()
|
||||
export class AuthClientService implements OnModuleInit {
|
||||
@@ -168,4 +169,63 @@ export class AuthClientService implements OnModuleInit {
|
||||
this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
|
||||
);
|
||||
}
|
||||
|
||||
async blockUsers(users: string[],metadata: Metadata): Promise<BulkEditResponse> {
|
||||
this.logger.info('blockUsers - Service starting');
|
||||
|
||||
try {
|
||||
this.logger.debug('Calling BlockUser gRPC method', {
|
||||
metadata: {
|
||||
access_token: metadata.get('access_token'),
|
||||
language: metadata.get('language')
|
||||
}
|
||||
});
|
||||
|
||||
const response = await lastValueFrom<BulkEditResponse>(
|
||||
this.authService.BlockUser({ users }, metadata),
|
||||
);
|
||||
|
||||
this.logger.info('blockUsers - Service success', { response });
|
||||
return response;
|
||||
} catch (error) {
|
||||
this.logger.error('blockUsers - Service error', {
|
||||
error: error.message,
|
||||
stack: error.stack,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async unblockUsers(users: string[],metadata: Metadata): Promise<BulkEditResponse> {
|
||||
this.logger.info('unblockUsers');
|
||||
return await lastValueFrom(
|
||||
this.authService.UnblockUser({ users }, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async resetUsers(users: string[], metadata: Metadata): Promise<BulkEditResponse> {
|
||||
this.logger.info('resetUsers');
|
||||
|
||||
try {
|
||||
this.logger.debug('Calling ResetUser gRPC method', {
|
||||
metadata: {
|
||||
access_token: metadata.get('access_token'),
|
||||
language: metadata.get('language')
|
||||
}
|
||||
});
|
||||
|
||||
const response = await lastValueFrom<BulkEditResponse>(
|
||||
this.authService.ResetUser({ users }, metadata)
|
||||
);
|
||||
|
||||
this.logger.info('resetUsers - Success', { response });
|
||||
return response;
|
||||
} catch (error) {
|
||||
this.logger.error('resetUsers - Error', {
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,16 +67,24 @@ export class AuthUser {
|
||||
export class AuthCustomer {
|
||||
@ApiProperty()
|
||||
modules: string[];
|
||||
|
||||
@ApiProperty()
|
||||
id: string;
|
||||
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
|
||||
@ApiProperty()
|
||||
displayName: string;
|
||||
@ApiProperty()
|
||||
tier: string;
|
||||
|
||||
@ApiProperty()
|
||||
scheduleLimit: string;
|
||||
|
||||
@ApiProperty()
|
||||
links: Link[];
|
||||
|
||||
@ApiProperty()
|
||||
themeEnabled: boolean;
|
||||
}
|
||||
@@ -120,3 +128,13 @@ export class AuthRefreshAccessTokenRes {
|
||||
@ApiProperty()
|
||||
accessToken: string;
|
||||
}
|
||||
|
||||
export interface BulkEditRequest {
|
||||
users: string[];
|
||||
}
|
||||
|
||||
export interface BulkEditResponse {
|
||||
message: string;
|
||||
successfulUsers: string[];
|
||||
failedUsers: string[];
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
} from '@nestjs/microservices';
|
||||
import { credentials } from '@grpc/grpc-js';
|
||||
import { Catalog } from '@dadosfera/protospack-v2';
|
||||
import { PlatformInterfaces } from '@dadosfera/protospack-v2';
|
||||
|
||||
const isLocalConnection =
|
||||
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
||||
@@ -19,11 +20,13 @@ export class CatalogClientConfiguration {
|
||||
package: [
|
||||
Catalog.ProtoPackages.ReadPackage,
|
||||
Catalog.ProtoPackages.WritePackage,
|
||||
PlatformInterfaces.ProtoPackages.WritePackage
|
||||
],
|
||||
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
||||
protoPath: [
|
||||
Catalog.ProtoPaths.ReadFilePath,
|
||||
Catalog.ProtoPaths.WriteFilePath,
|
||||
PlatformInterfaces.ProtoPaths.WriteFilePath
|
||||
],
|
||||
loader: {
|
||||
keepCase: true,
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
Put,
|
||||
Query,
|
||||
UseFilters,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
@@ -30,6 +32,7 @@ import { CatalogService } from './catalog.service';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import {
|
||||
BatchRemoveRlsRulesRequest,
|
||||
GetDatasetCatalogTaskRes,
|
||||
@@ -54,6 +57,7 @@ import {
|
||||
AddRlsRuleRequest,
|
||||
GetNimbusDashboardsRequest,
|
||||
GetRlsRulesRequest,
|
||||
RegisterDatasetWithMetatadaRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||
|
||||
@ApiTags('Catalog')
|
||||
@@ -743,4 +747,94 @@ export class CatalogController {
|
||||
|
||||
return JSON.parse(dashboards);
|
||||
}
|
||||
|
||||
|
||||
@Post('register-dataset')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
|
||||
)
|
||||
async registerDatasetWithMetadataRequest(
|
||||
@Body() body: RegisterDatasetWithMetatadaRequest,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const logMetadata = {
|
||||
customer_name: customer_name,
|
||||
user_id: user_id,
|
||||
method: 'POST',
|
||||
path: '/catalog/register-dataset',
|
||||
};
|
||||
try {
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
});
|
||||
|
||||
this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
|
||||
|
||||
// Create table metadata
|
||||
const tableMetadataBody = {
|
||||
table_metadata: body.table_metadata,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
|
||||
},
|
||||
logMetadata: logMetadata
|
||||
};
|
||||
|
||||
const table_metadata_id = await this.catalogService.createTableMetadata(
|
||||
tableMetadataBody,
|
||||
);
|
||||
this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
|
||||
|
||||
// Create column metadata
|
||||
const columnMetadataBody = {
|
||||
column_metadata: body.column_metadata,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
|
||||
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
|
||||
|
||||
// Create data preview
|
||||
const dataPreviewBody = {
|
||||
data_preview: body.data_preview,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
|
||||
const data_preview_id = await this.catalogService.createDataPreview(
|
||||
dataPreviewBody,
|
||||
);
|
||||
|
||||
// Catalog dataset item
|
||||
this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
|
||||
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
|
||||
this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
|
||||
return {
|
||||
message: 'Dataset registered successfully',
|
||||
table_metadata_id: table_metadata_id,
|
||||
column_metadata_ids: column_metadata_ids,
|
||||
data_preview_id: data_preview_id,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
|
||||
|
||||
throw new HttpException(
|
||||
{
|
||||
message: 'Ocorreu um erro ao registrar o dataset',
|
||||
error: error.message,
|
||||
code: 'REGISTRATION_FAILED',
|
||||
details: error.message,
|
||||
},
|
||||
HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,11 @@ import {
|
||||
ProtoServices,
|
||||
Messages,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
|
||||
import {
|
||||
Messages as PlatformInterfaceMessages,
|
||||
WriteService as PlatformInterfaceWriteService,
|
||||
ProtoServices as PlatformInterfacesProtoServices,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
|
||||
import {
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
@@ -33,6 +38,7 @@ import {
|
||||
class CatalogService implements OnModuleInit {
|
||||
catalogReadService: ReadService.CatalogReadServices;
|
||||
catalogWriteService: WriteService.CatalogWriteServices;
|
||||
platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
|
||||
logger: any;
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
@@ -54,9 +60,14 @@ class CatalogService implements OnModuleInit {
|
||||
this.grpcClient.getService<WriteService.CatalogWriteServices>(
|
||||
ProtoServices.CatalogWriteServices,
|
||||
);
|
||||
this.platformWriteService =
|
||||
this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
|
||||
PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
|
||||
);
|
||||
}
|
||||
|
||||
_getNimbusUrl(body) {
|
||||
this.logger.debug(`Body: ${JSON.stringify(body)}`);
|
||||
const customer = body.info.customer.toLowerCase();
|
||||
|
||||
if (process.env.ENV === 'prd') {
|
||||
@@ -420,6 +431,112 @@ class CatalogService implements OnModuleInit {
|
||||
);
|
||||
return res.dashboards;
|
||||
}
|
||||
|
||||
async createTableMetadata(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
|
||||
|
||||
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
|
||||
|
||||
this.logger.info(
|
||||
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
|
||||
{...body.logMetadata},
|
||||
);
|
||||
return data.id;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
|
||||
error.response?.status
|
||||
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async createColumnMetadata(body: any): Promise<number[]> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
|
||||
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
|
||||
|
||||
|
||||
|
||||
try {
|
||||
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
|
||||
const { data, status } = await axios.post(endpoint, body.column_metadata);
|
||||
|
||||
this.logger.info(
|
||||
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
|
||||
{...body.logMetadata},
|
||||
);
|
||||
return data.map((column) => column.id);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
|
||||
error.response?.status
|
||||
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async createDataPreview(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
|
||||
|
||||
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, body.data_preview);
|
||||
|
||||
this.logger.info(
|
||||
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
|
||||
{...body.logMetadata},
|
||||
);
|
||||
return data.id;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
|
||||
error.response?.status
|
||||
} because of ${error.response?.data || error.message}`,
|
||||
{...body.logMetadata},
|
||||
);
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||
const customer_name_raw = metadata.get('customer_name');
|
||||
|
||||
const customer_name = customer_name_raw?.[0]?.toString();
|
||||
if (!customer_name) {
|
||||
throw new BadRequestException('Customer name not found in metadata');
|
||||
}
|
||||
const res = await lastValueFrom(
|
||||
this.platformWriteService.CatalogDataAssets(
|
||||
{
|
||||
data_assets: [
|
||||
{
|
||||
data_asset_id: table_metadata_id.toString(),
|
||||
customer_name: customer_name,
|
||||
data_asset_type: 'dataset',
|
||||
},
|
||||
],
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
export { CatalogService };
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
|
||||
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
|
||||
import { init } from 'mixpanel';
|
||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||
@@ -13,7 +13,12 @@ export class MixpanelController {
|
||||
private readonly mixpanelToken: string,
|
||||
) {}
|
||||
@Post(':id')
|
||||
async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
|
||||
async trackEvent(
|
||||
@Param('id') id,
|
||||
@Body() body,
|
||||
@User() user: RequestUser,
|
||||
@Req() request
|
||||
) {
|
||||
delete body.info;
|
||||
const mixpanel = init(this.mixpanelToken);
|
||||
|
||||
@@ -66,13 +71,16 @@ export class MixpanelController {
|
||||
$email: user.username.includes('@')
|
||||
? user.username
|
||||
: user.username + '@dadosfera.ai',
|
||||
customer_name: user.customer_name,
|
||||
customer_name: user.customer_name
|
||||
});
|
||||
|
||||
await mixpanel.track(id, {
|
||||
distinct_id: user.username,
|
||||
customer: user.customer_name,
|
||||
env: process.env.ENV,
|
||||
$ip: request.ip,
|
||||
$os: request.headers['sec-ch-ua-platform'] || '',
|
||||
$browser: request.headers['user-agent'],
|
||||
...body,
|
||||
});
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
import { Body, Controller, ForbiddenException, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
|
||||
import { Body, Controller, ForbiddenException, Header, Headers, HttpCode, HttpException, Inject, Param, Post, Query, Req, Res, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
|
||||
import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
|
||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
@@ -40,20 +40,23 @@ export class OpenDataController {
|
||||
language: string,
|
||||
@Req()
|
||||
request: Request,
|
||||
@Headers('origin')
|
||||
origin: string
|
||||
) {
|
||||
if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) {
|
||||
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
|
||||
|
||||
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
|
||||
if (!corslist.includes(origin)) {
|
||||
this.logger.info('block request by cors list: '+ origin);
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
this.logger.info('createUser for open data');
|
||||
|
||||
this.logger.info('user-agent', request.headers['user-agent']);
|
||||
|
||||
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
|
||||
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
|
||||
const roles = [process.env.OPEN_GROUP_ID];
|
||||
const metadata = PackTheMetadata({
|
||||
language: language || 'en-us'
|
||||
});
|
||||
});
|
||||
|
||||
const data = {}
|
||||
|
||||
|
||||
@@ -77,6 +77,13 @@ export function EnrichErrorCode(code: string) {
|
||||
message: 'Token para recuperar senha inválido',
|
||||
code,
|
||||
};
|
||||
case ErrorCodes.AUTH.REQUIRED_PASSWORD_RESET:
|
||||
return {
|
||||
statusCode: HttpStatus.UNAUTHORIZED,
|
||||
error: 'Não permitido',
|
||||
message: 'Seu usuário foi resetado por um administrador, resete a senha pelo link "Esqueci a senha", para prosseguir',
|
||||
code,
|
||||
};
|
||||
|
||||
case ErrorCodes.AUTH.WEAK_NEW_PASSWORD:
|
||||
return {
|
||||
|
||||
@@ -13,6 +13,7 @@ export const AUTH = {
|
||||
RESET_PASSWORD_CODE_EXPIRED: 'AUTH.RESET_PASSWORD_CODE_EXPIRED',
|
||||
RESET_PASSWORD_CODE_INVALID: 'AUTH.RESET_PASSWORD_CODE_INVALID',
|
||||
WEAK_NEW_PASSWORD: 'AUTH.WEAK_NEW_PASSWORD',
|
||||
REQUIRED_PASSWORD_RESET: 'AUTH.REQUIRED_PASSWORD_RESET'
|
||||
};
|
||||
export const ROLE = {
|
||||
INVALID_ID: 'ROLE.INVALID_ID',
|
||||
|
||||
Reference in New Issue
Block a user