gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-08-31 19:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'fix/redis-tls-configurable' of github.com:dadosfera/maestro into fix/redis-tls-configurable

Browse Source
This commit is contained in:
Rafael
2025-12-16 16:57:59 -03:00
parent 0a5e8001f9 b2700d4bb0
commit 295f1f86ca
20 changed files with 676 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/k8s-deploy.yml
+3 -3
View File
@@ -56,9 +56,9 @@ jobs:
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
@@ -105,7 +105,7 @@ jobs:
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
.github/workflows/validate-k8s.yml
+6 -3
View File
@@ -66,13 +66,16 @@ jobs:
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Debug Helm env
run: |
helm env
echo "HOME=$HOME"
ls -R $HOME/.local/share/helm || true
- name: Authenticate with OKE cluster
env:
deploy/helm-chart/templates/ingress-private.yaml
+9
View File
@@ -4,9 +4,18 @@ metadata:
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
docsfera.json
+101
View File
@@ -3877,6 +3877,36 @@
],
"type": "string"
}
},
{
"name": "owner",
"required": false,
"in": "query",
"description": "ID do usuário owner para filtrar data assets",
"example": "user-id-1,user-id-2",
"schema": {
"type": "string"
}
},
{
"name": "catalog_date_from",
"required": false,
"in": "query",
"description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
"example": "2025-01-01",
"schema": {
"type": "string"
}
},
{
"name": "catalog_date_to",
"required": false,
"in": "query",
"description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
"example": "2025-12-31",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -4006,6 +4036,36 @@
],
"type": "string"
}
},
{
"name": "owner",
"required": false,
"in": "query",
"description": "ID do usuário owner para filtrar data assets",
"example": "user-id-1,user-id-2",
"schema": {
"type": "string"
}
},
{
"name": "catalog_date_from",
"required": false,
"in": "query",
"description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
"example": "2025-01-01",
"schema": {
"type": "string"
}
},
{
"name": "catalog_date_to",
"required": false,
"in": "query",
"description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
"example": "2025-12-31",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -4471,6 +4531,14 @@
"schema": {
"type": "string"
}
},
{
"name": "asset_type",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
@@ -6269,6 +6337,39 @@
]
}
},
"/customers/{id}/theme/reset": {
"post": {
"operationId": "ThemeController_resetTheme",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerThemeResponse"
}
}
}
},
"201": {
"description": ""
}
},
"tags": [
"Theme"
]
}
},
"/network-policy": {
"get": {
"operationId": "NetworkPolicyController_getNetworks",
package-lock.json
Generated
+4 -5
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.16",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -2970,10 +2970,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.10.tgz",
"integrity": "sha512-4miwovVFHtBY1VTHdW8BVmSB8m+LXfIA8uigHbeo8IwreMtTHuLpvfenT4MBSPkdVmlo9+0XBKKf+PaSqtdMAg==",
"license": "ISC",
"version": "3.38.0-beta.16",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.16.tgz",
"integrity": "sha512-ijURcmETXdgVgXhIPxapK7w0z6begXm4WDZ6EJtUGg+oYlJd9QOxbwaD7akRzW8XTdccmIo0Vov065KB812J0A==",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
package.json
+1 -1
View File
@@ -35,7 +35,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.16",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
src/authentication/extract-user.ts
+20
View File
@@ -0,0 +1,20 @@
import jwt, { JwtPayload } from 'jsonwebtoken';
export function extractUserFrom(aRawJwt: string) {
const decodedToken = jwt.decode(aRawJwt, {
complete: true,
});
const payload = decodedToken.payload as JwtPayload;
return {
user_id: payload.user_id,
username: payload.username,
permissions: payload.permissions,
customer_id: payload.customer_id,
customer_name: payload.customer_name,
customer_tier: payload.customer_tier,
customer_modules: payload.customer_modules,
access_token: aRawJwt,
}
}
src/authentication/permissions.enum.ts
+19
View File
@@ -390,6 +390,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
LINEAGE: {
title: {
'pt-br': 'Explorar | Linhagem',
'en-us': 'Explore | Lineage',
'es-es': 'Explorar | Linaje',
},
permissions: {
VIEW: {
seqid: 50,
claim: 'lineage:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar ao módulo de Linhagem',
'en-us': 'Access to Lineage module',
'es-es': 'Acceda al módulo de Linaje',
},
}
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
src/main.ts
+19 -9
View File
@@ -18,26 +18,36 @@ async function bootstrap() {
});
const logger = new DadosferaLogger();
const corsOrigins = [];
if (process.env.ENV === 'local') {
corsOrigins.push('http://localhost:4200');
} else {
corsOrigins.push(
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://private-frontend.stg.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
);
}
const app = await NestFactory.create(AppModule, {
logger,
cors: {
origin: [
'http://localhost:4200',
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
],
origin: corsOrigins,
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true,
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
if (process.env.ENV !== 'local') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use(
'/catalog/register-dataset',
src/modules/assign/assign.controller.ts
+6 -2
View File
@@ -1,4 +1,4 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
@@ -28,6 +28,10 @@ export class AssignController {
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
try {
return await this.assignService.get(metadata);
} catch (error) {
throw new NotFoundException(error.message)
}
}
}
src/modules/auth/auth.controller.ts
+30 -127
View File
@@ -13,6 +13,7 @@ import {
Req,
Param,
Res,
UnauthorizedException,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -54,7 +55,6 @@ import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
@@ -103,14 +103,13 @@ export class AuthController {
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
this.authClient.writeAuthSession(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
@@ -127,27 +126,8 @@ export class AuthController {
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
this.authClient.cleanUpAuthSession(res);
return res.send();
} catch (error) {
@@ -176,8 +156,9 @@ export class AuthController {
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
this.authClient.writeAuthSession(res, {
accessToken: data.accessToken,
refreshToken: data.refreshToken,
userId
});
@@ -493,114 +474,36 @@ export class AuthController {
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
// Get token and headers
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
const refreshToken = req.cookies['ddf-refresh-auth'];
const userId = req.cookies['ddf-user-id'];
const resourceHost = req.headers["host"]
const hasUserSession = Boolean(accessToken) && Boolean(userId);
this.logger.info('Has User Session: ' + hasUserSession);
if (!hasUserSession) {
throw new UnauthorizedException()
}
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
const userDto = await this.authClient.validateUserSession(accessToken, resourceHost);
return res.status(200).json(userDto);
} catch (error) {
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
if (!refreshToken) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
throw new UnauthorizedException("Invalid refresh token or customer name");
};
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
const {
authSession,
user
} = await this.authClient.refreshUserSession(refreshToken, userId, resourceHost);
this.authClient.writeAuthSession(res, authSession);
return res.status(200).json(user);
}
}
}
src/modules/auth/auth.service.ts
+223 -17
View File
@@ -1,10 +1,20 @@
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import {
OnModuleInit,
Inject,
Injectable,
ForbiddenException,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthProtoService as AuthServiceInterface,
UsersProtoService,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -21,18 +31,24 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
import { BulkEditResponse, UserDTO } from './dtos/login';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { Request, Response } from 'express';
type AuthSession = {
accessToken?: string;
refreshToken?: string;
userId?: string;
};
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
private userService: UsersProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -46,8 +62,8 @@ export class AuthClientService implements OnModuleInit {
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
this.userService = this.grpcClient.getService<UsersProtoService>(
ProtoServices.UsersProtoService,
);
}
@@ -63,11 +79,11 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
checkDedicatedProxy({ customer }: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info(
'SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY,
);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
@@ -75,7 +91,9 @@ export class AuthClientService implements OnModuleInit {
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
this.logger.info(
'Check if customer have network policy: ' + customer.modules,
);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
@@ -94,7 +112,6 @@ export class AuthClientService implements OnModuleInit {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
@@ -105,7 +122,7 @@ export class AuthClientService implements OnModuleInit {
this.checkDedicatedProxy(result);
}
return result
return result;
}
async refreshAccessToken(
@@ -115,7 +132,10 @@ export class AuthClientService implements OnModuleInit {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
this.authService.AuthRefreshAccessToken(
{ refreshToken, userId },
metadata,
),
);
}
@@ -283,4 +303,190 @@ export class AuthClientService implements OnModuleInit {
throw error;
}
}
public async validateUserSession(accessToken: any, resourceHost: string) {
const payload = await this.validateJwtToken(accessToken);
const userDto = await this.getUserfromPayload(payload);
this.validateResourceAccess(resourceHost, userDto);
return userDto;
}
public async refreshUserSession(
refreshToken: string,
userId: string,
originHeader: string,
): Promise<{
user: UserDTO;
authSession: AuthSession;
}> {
const metadata = PackTheMetadata({});
this.logger.info('Call Refresh Token');
const refreshCredentials = await this.refreshAccessToken(
{ refreshToken, userId },
metadata,
);
this.logger.info('Finish Refresh Token');
const userDto = await this.validateUserSession(
refreshCredentials.accessToken,
originHeader,
);
return {
user: userDto,
authSession: {
accessToken: refreshCredentials.accessToken,
refreshToken: refreshCredentials.refreshToken,
userId,
},
};
}
public writeAuthSession(res: Response, data: AuthSession) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth');
res.cookie('ddf-auth', data.accessToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth');
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth');
res.cookie('ddf-user-id', data.userId, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
public cleanUpAuthSession(res: Response) {
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.ai',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.ai',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
}
private async validateJwtToken(token: string) {
const decoded: any = token && jwt.decode(token, { complete: true });
if (!decoded) throw new Error('Invalid token');
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(token, pemValue);
return decoded.payload;
}
private async getUserfromPayload(payload: JwtPayload): Promise<UserDTO> {
this.logger.info('getUser');
const metadata = PackTheMetadata({
customer_id: payload.customer_id,
});
const { user } = await lastValueFrom(
this.userService.UserFindOneById({ id: payload.user_id }, metadata),
);
const userDto: UserDTO = {
id: user.id,
name: user.username,
jobTitle: user?.jobTitle || null,
department: user?.department || null,
hierarchy: user?.hierarchy || null,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
},
};
return userDto;
}
private validateResourceAccess(host: string, user: UserDTO) {
this.logger.info(
"Validate whether the source URL is a resource belonging to the user's client",
);
this.logger.info('Host: ' + host);
this.logger.info('Customer: ' + user.customer.name);
const hostParts = host.split('.');
const domain = hostParts[0];
const isResouceStg = hostParts[1] === 'stg';
const notFoundCustomerInDomain = !domain.includes('-')
if (notFoundCustomerInDomain) {
this.logger.info(`Not found Customer Name in domain`);
return;
}
const domainParts = domain.split('-');
const customerInDomain = domainParts[domainParts.length - 1];
if (isResouceStg && process.env.ENV !== 'stg') {
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
throw new HttpException(
`Customer ${user.customer.name} cannot access ${host}`,
HttpStatus.FORBIDDEN
);
}
if (customerInDomain != user.customer.name) {
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
throw new HttpException(
`Customer ${user.customer.name} cannot access ${host}`,
HttpStatus.FORBIDDEN
);
}
return;
}
}
src/modules/auth/dtos/login.ts
+13
View File
@@ -140,3 +140,16 @@ export interface BulkEditResponse {
successfulUsers: string[];
failedUsers: string[];
}
export type UserDTO = {
id: string,
name: string,
jobTitle?: string,
department?: string,
hierarchy?: string,
customer: {
id: string,
name: string,
tier: string,
}
}
src/modules/catalog/catalog.controller.ts
+17 -5
View File
@@ -423,6 +423,7 @@ export class CatalogController {
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Query('asset_type') asset_type: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -439,7 +440,7 @@ export class CatalogController {
language,
});
const docs = await this.catalogService.getDataDocs(id, metadata);
const docs = await this.catalogService.getDataDocs(id, asset_type, metadata);
return { docs };
}
@@ -487,21 +488,32 @@ export class CatalogController {
@Headers() headers,
@Param('id') table_id: string,
@Body('docs') docs: string,
@Query('asset_type') asset_type: string,
) {
const { user_id, customer_name } = user;
const { user_id, customer_name, customer_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
this.logger.info(`/catalog - ON POST DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const res = await this.catalogService.createDataDocs({
const body = {
table_id,
docs,
asset_type,
info: {
customer: customer_name,
},
});
}
const res = await this.catalogService.createDataDocs(body, metadata);
return res;
}
src/modules/catalog/catalog.service.ts
+120 -8
View File
@@ -24,9 +24,11 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
CreateDataDocsDTO,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
@@ -39,6 +41,7 @@ import {
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
catalogWriteService: WriteService.CatalogWriteServices;
@@ -55,6 +58,7 @@ class CatalogService implements OnModuleInit {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
@@ -70,20 +74,24 @@ class CatalogService implements OnModuleInit {
);
}
_getNimbusUrl(body) {
this.logger.debug(`Body: ${JSON.stringify(body)}`);
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
return `https://nimbus-${customer}.dadosfera.ai`;
}
return `https://nimbus-${customer}.${process.env.ENV.replace(
'local',
'stg',
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
try {
@@ -94,8 +102,10 @@ class CatalogService implements OnModuleInit {
)
this.logger.info("Finish grpc call")
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
@@ -105,9 +115,11 @@ class CatalogService implements OnModuleInit {
'pdf': 'application/pdf'
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
@@ -118,20 +130,25 @@ class CatalogService implements OnModuleInit {
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
return lastValueFrom(
this.catalogWriteService.CreateDataAsset(data, metadata),
);
}
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.ManagePermission(data, metadata),
).catch((err) => {
@@ -142,9 +159,11 @@ class CatalogService implements OnModuleInit {
});
}
async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.RevokePermission(data, metadata),
).catch((err) => {
@@ -155,9 +174,11 @@ class CatalogService implements OnModuleInit {
});
}
async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.MakeAComment(data, metadata),
).catch((err) => {
@@ -168,9 +189,11 @@ class CatalogService implements OnModuleInit {
});
}
async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.UpdateAComment(data, metadata),
).catch((err) => {
@@ -181,9 +204,11 @@ class CatalogService implements OnModuleInit {
});
}
async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.DeleteDataAsset(data, metadata),
).catch((err) => {
@@ -194,14 +219,20 @@ class CatalogService implements OnModuleInit {
});
}
async getUserRolesIds(userId: string) {
const result = await this.userService.findOneById(userId).catch(() => null);
const roles_ids = result.user.roles.map((role) => role.id);
return roles_ids;
if (result) {
return result.user.roles.map((role) => role.id);
}
return [];
}
async searchDataAssets(
query: Record<string, any>,
metadata: Metadata,
@@ -209,6 +240,7 @@ class CatalogService implements OnModuleInit {
) {
this.logger.info('CatalogService - searchDataAssets');
const { search, page, size, sort_by, order, ...filters } = query;
const { data_assets, total } = await lastValueFrom(
@@ -225,16 +257,20 @@ class CatalogService implements OnModuleInit {
),
);
const result = JSON.parse(data_assets);
const response = await this.getAssetsUsersAndRoles(
result.data_assets,
customer_id,
);
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
@@ -242,6 +278,7 @@ class CatalogService implements OnModuleInit {
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
id: asset.id,
display_name: asset.display_name,
@@ -250,19 +287,24 @@ class CatalogService implements OnModuleInit {
tags: '[' + asset.tags.join(', ') + ']'
}))
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -283,9 +325,11 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getOneDataAssetByPipelineAndObject(data: {
customer_id: string;
pipeline: string;
@@ -307,9 +351,11 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
@@ -318,6 +364,7 @@ class CatalogService implements OnModuleInit {
}) {
const { body, customer_id, data_asset_id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{ id: data_asset_id, changes: JSON.stringify(body) },
@@ -332,18 +379,20 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, metadata: Metadata) {
async getDataDocs(id: string, assetType: string, metadata: Metadata) {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
this.catalogReadService.GetDatasetDoc({ id }, metadata),
);
const docs = JSON.parse(documentation);
return docs;
}
async getDatasetPreview(id: string, metadata: Metadata) {
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
@@ -355,6 +404,7 @@ class CatalogService implements OnModuleInit {
return result;
}
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
@@ -366,7 +416,16 @@ class CatalogService implements OnModuleInit {
return result;
}
async createDataDocs(body) {
async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
if (body.asset_type === 'table' || body.asset_type === 'view') {
return this.createDataDocsViaNimbus(body);
}
return this.createDataDocsViaGrpc(body, metadata);
}
private async createDataDocsViaNimbus(body: CreateDataDocsDTO) {
this.logger.info('Creating data docs via Nimbus for table/view');
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-docs/`,
@@ -375,9 +434,35 @@ class CatalogService implements OnModuleInit {
return data;
}
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
this.logger.info('Creating data docs via gRPC for other asset types');
try {
const response: any = await lastValueFrom(
this.catalogWriteService.UpdateDataAssetDoc(
{
id: body.table_id,
docs: body.docs,
},
metadata,
),
);
return response;
} catch (error) {
this.logger.error('Error creating data asset docs:', error);
throw new HttpException(
'Failed to create data asset documentation',
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
const response = await lastValueFrom(
this.catalogReadService.GetCustomerTags(data, metadata),
)
@@ -390,6 +475,7 @@ class CatalogService implements OnModuleInit {
throw new Error(err);
});
return response;
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
@@ -402,7 +488,8 @@ class CatalogService implements OnModuleInit {
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
)?.email;
const roles = [];
const users = [];
@@ -412,10 +499,11 @@ class CatalogService implements OnModuleInit {
if (role) roles.push({ id: role.id, name: role.name });
}
const data_asset_users = data_asset?.users || []
for (const user_id of data_asset_users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
if (user) users.push({ id: user.id, email: user.email });
}
return {
...data_asset,
@@ -426,6 +514,7 @@ class CatalogService implements OnModuleInit {
});
}
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
const { session } = await lastValueFrom(
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -439,6 +528,7 @@ class CatalogService implements OnModuleInit {
return res;
}
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.AddRlsRule(data, metadata),
@@ -446,6 +536,7 @@ class CatalogService implements OnModuleInit {
return res;
}
async removeRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
@@ -453,12 +544,14 @@ class CatalogService implements OnModuleInit {
return res;
}
async batchRemoveRlsRule(
query: BatchRemoveRlsRulesRequest,
metadata: Metadata,
) {
const { id_rls, nimbus_dashboard_id } = query;
if (id_rls && nimbus_dashboard_id) {
throw new BadRequestException(
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
@@ -479,6 +572,7 @@ class CatalogService implements OnModuleInit {
return 'OK';
}
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetRlsRules(data, metadata),
@@ -486,6 +580,7 @@ class CatalogService implements OnModuleInit {
return res.rls_rules;
}
async getOneRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetOneRlsRule({ id }, metadata),
@@ -493,6 +588,7 @@ class CatalogService implements OnModuleInit {
return res.rls_rule;
}
async getNimbusDashboards(
data: GetNimbusDashboardsRequest,
metadata: Metadata,
@@ -503,19 +599,24 @@ class CatalogService implements OnModuleInit {
return res.dashboards;
}
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{...body.logMetadata},
@@ -530,19 +631,23 @@ class CatalogService implements OnModuleInit {
}
}
async createColumnMetadata(body: any): Promise<number[]> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{...body.logMetadata},
@@ -557,18 +662,22 @@ class CatalogService implements OnModuleInit {
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{...body.logMetadata},
@@ -585,9 +694,11 @@ class CatalogService implements OnModuleInit {
}
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
const customer_name = customer_name_raw?.[0]?.toString();
if (!customer_name) {
throw new BadRequestException('Customer name not found in metadata');
@@ -610,4 +721,5 @@ class CatalogService implements OnModuleInit {
}
}
export { CatalogService };
export { CatalogService };
src/modules/catalog/dtos/index.ts
+25
View File
@@ -147,6 +147,24 @@ export class ICatalogAllRequest {
description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
})
order?: OrderEnum;
@ApiPropertyOptional({
description: 'ID do usuário owner para filtrar data assets',
example: 'user-id-1,user-id-2',
})
owner?: string;
@ApiPropertyOptional({
description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)',
example: '2025-01-01',
})
catalog_date_from?: string;
@ApiPropertyOptional({
description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)',
example: '2025-12-31',
})
catalog_date_to?: string;
}
export class ICatalogAllResponse {
@@ -328,3 +346,10 @@ export type AssetReporter = {
created_at: string;
tags: string;
}
export type CreateDataDocsDTO = {
table_id: string;
docs: string;
asset_type: string;
}
src/modules/catalog/share/share.service.ts
+3 -3
View File
@@ -160,7 +160,7 @@ export class ShareService implements OnModuleInit {
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
this.catalogReadService.GetDatasetDoc({ id }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
@@ -180,7 +180,7 @@ export class ShareService implements OnModuleInit {
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
)?.email;
const roles = [];
const users = [];
@@ -190,7 +190,7 @@ export class ShareService implements OnModuleInit {
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
if (user) users.push({ id: user.id, email: user.email });
}
return {
...data_asset,
src/modules/mixpanel/mixpanel.controller.ts
+25 -8
View File
@@ -1,29 +1,46 @@
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
import { extractUserFrom } from 'src/authentication/extract-user';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@ApiInternalOnlyController()
@Controller('trackEvent')
export class MixpanelController {
logger: DadosferaLogger;
constructor(
private mixpanelService: MixpanelService
) {}
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private mixpanelService: MixpanelService,
) {
this.logger = dadosferaLogger.logger;
}
@Post(':id')
async trackEvent(
@Param('id') id,
@Body() body,
@User() user: RequestUser,
@Req() request
) {
this.logger.info(`POST Track Event: ${id}`)
delete body.info;
const anonymousUser = {
username: "anonymous",
customer_name: "anonymous"
} as RequestUser
const hasToken = request.headers['authorization'];
const user = hasToken ? extractUserFrom(hasToken) : anonymousUser;
this.logger.info(`Has user: ${typeof hasToken == "string"}`)
await this.mixpanelService.track(id, user, request, body)
this.logger.info(`Event successful`)
return { id, body, user: user.username };
}
}
src/modules/theme/theme.controller.ts
+20
View File
@@ -124,4 +124,24 @@ export class ThemeController {
}
}
@Post('/:id/theme/reset')
@ApiOkResponse({ type: CustomerThemeResponse })
async resetTheme(@Param('id') id: string) {
this.logger.info('getCustomerTheme with id' + id);
try {
await this.themeService.resetTheme(id);
return { theme: null };
}catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND');
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
} else {
this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message);
return { theme: null };
};
}
}
}
src/modules/theme/theme.service.ts
+12
View File
@@ -44,6 +44,18 @@ export class ThemeService implements OnModuleInit {
);
}
async resetTheme(id: string) {
const { theme } = await firstValueFrom(
this.themeService.ResetCustomerTheme({
id
}),
);
return {
theme
}
}
async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) {
if (!id) {
this.logger.error('Error - saveCustomertheme - not found id:' + id);