Compare commits

...
Author SHA1 Message Date
viniciusgadea 3ab2f8f27d CHORE: remove linkFileToTable endpoint from Storage Explorer controller 2026-01-14 08:51:45 -03:00
viniciusgadea b44d23552c CHORE: remove unecessary header for request 2026-01-14 06:40:52 -03:00
viniciusgadea 8e63757738 FEAT: add STORAGE_EXPLORER_API_URL to deployment and values configuration 2026-01-14 06:26:21 -03:00
viniciusgadea 3b84409003 CHORE: add unique authentication decorator to Storage Explorer controller methods 2026-01-14 06:25:58 -03:00
viniciusgadea 0ce3822300 CHORE: update new endpoint link-table and remove /delete from storage-explorer api 2026-01-13 14:36:43 -03:00
viniciusgadea a2c7ce00db FEAT: integrate Storage Explorer API with new endpoints and permissions 2026-01-13 10:29:38 -03:00
Rafael Santana 53920f2f3f Merge pull request #431 from dadosfera/feature/mgc-keycloak-migration
Feature/mgc keycloak migration
2026-01-08 18:21:10 -03:00
RafaelandClaude Opus 4.5 8c34914806 FEAT: add AUTH_PROVIDER config to Helm chart
- Add auth_provider to values.yaml (cognito/keycloak)
- Add AUTH_PROVIDER env var to deployment template
- Note: maestro only needs to know which provider is used, duc handles connection

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-08 12:22:47 -03:00
RafaelandClaude Opus 4.5 1881d07c4a FEAT: add dual auth provider support to Helm chart
- Add auth_provider config to values.yaml (cognito/keycloak)
- Add AUTH_PROVIDER env var to deployment template
- Add conditional Keycloak env vars (URL, realm, client ID, secret)
- Enables parallel deployments with different auth providers

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-08 12:10:17 -03:00
RafaelandClaude Opus 4.5 3b8310fdca CHORE: simplify Dockerfile.local to use npm ci only
Remove protospack tarball fallback logic

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-07 19:04:07 -03:00
RafaelandClaude Opus 4.5 52bda8ebe2 FEAT: add authProvider field to user response
- Add authProvider to user entity DTO
- Update user service to include authProvider
- Update auth controller response

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-07 18:55:36 -03:00
Rafael 275a53dbd1 Merge remote-tracking branch 'origin/beta' into feature/mgc-keycloak-migration 2026-01-06 12:44:05 -03:00
vinicius gadea 9cefdb226d Merge pull request #430 from dadosfera/feat/organization-form
Feat/organization form
2026-01-06 10:52:33 -03:00
viniciusgadea 8ac0a8a79f REFACTOR: rename 'companySite' to 'personalSite' in user DTOs and related services 2026-01-05 15:43:35 -03:00
viniciusgadea 285de97375 FIX: update organization info field from 'name' to 'companyName' in updateOrganizationInfo method 2026-01-05 12:01:04 -03:00
viniciusgadea e2a7d2b92b FIX: update organization info field from 'name' to 'companyName' 2026-01-05 11:47:20 -03:00
viniciusgadea da23ad76db DOCS: update docs to organization forms 2026-01-02 09:22:27 -03:00
viniciusgadea 55b0961b82 FEAT: add organization info endpoints and DTOs for update and retrieval new organization forms 2026-01-02 09:21:46 -03:00
Rafael Santana d3c5c0fa63 Merge pull request #427 from dadosfera/fix/security-vulnerabilities-high
FIX: resolve HIGH security vulnerabilities (18 → 0)
2025-12-29 11:47:51 -03:00
RafaelandClaude Opus 4.5 5dbc644d1d fix: security vulnerability - upgrade multer to 2.0.2
- Update multer override from 1.4.5-lts.1 to 2.0.2 to fix HIGH vulnerabilities:
  - CVE-2025-47935 (DoS via memory leaks)
  - CVE-2025-47944
  - CVE-2025-48997
  - CVE-2025-7338

Reduces HIGH vulnerabilities from 4 to 0

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-29 11:42:04 -03:00
Marcos Rodrigues Silva 8eddb9e1bf Merge pull request #428 from dadosfera/hotfix/user-patch
Hotfix/user patch
2025-12-26 17:58:28 -03:00
Marcos Rodrigues 76485f929d FIX: user patch route 2025-12-26 17:53:02 -03:00
RafaelandClaude Opus 4.5 8e0182aa50 FEAT: add TOTP support for change password and local build support
- Pass TOTP code to DUC for Keycloak users with MFA
- Add Dockerfile.local for local protospack builds
- Add .dockerignore to exclude node_modules from Docker context

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-24 19:03:50 -03:00
Rafael Santana a18bdccc09 Merge branch 'beta' into fix/security-vulnerabilities-high 2025-12-23 13:51:25 -03:00
RafaelandClaude b27298501d FIX: resolve HIGH security vulnerabilities (18 → 0)
Upgraded Node.js and dependencies to address critical vulnerabilities:

**Infrastructure:**
- Upgrade Node.js from 18.17-alpine to 20-alpine (Alpine 3.18 → 3.23)
- Upgrade npm to 11.7.0 (includes glob 13.0.0 fix)
- Replace npm ci with npm install, then back to npm ci with npm 11 lock

**Dependencies:**
- Update axios: 0.27.2 → 0.30.2 (fixes 2 CVEs)
- Update body-parser: 1.20.1/1.20.2 → 1.20.3
- Update cross-spawn: 7.0.3 → 7.0.6 (hoisted)
- Update glob: 10.2.4 → 10.5.0 (hoisted)
- Update path-to-regexp: 3.2.0 → 3.3.0 (maintains @nestjs/swagger compatibility)
- Update semver: 7.5.1 → 7.5.2

**Results:**
- HIGH vulnerabilities: 18 → 0 (100% reduction)
- Alpine OS vulnerabilities: 6 → 0 (fixed by Alpine 3.23)
- Node.js vulnerabilities: 12 → 0
- All tests passing
- Production verified working

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-12-23 13:48:02 -03:00
vinicius gadea 33ebc91826 Merge pull request #426 from dadosfera/feature/catalog-update
FEAT: Add optional 'docs' field to assets
2025-12-19 08:24:54 -03:00
vinicius gadea 6948156693 Merge branch 'beta' into feature/catalog-update 2025-12-19 08:21:52 -03:00
viniciusgadea 0aaa4384c3 DOCS: Add 'docs' field to schemas in docsfera.json and update protospack-v2 to version 3.38.0-beta.19 in package-lock.json 2025-12-18 17:43:54 -03:00
Marcos Rodrigues Silva 1f9d0c29ec Merge pull request #425 from dadosfera/beta
Beta
2025-12-18 17:42:10 -03:00
viniciusgadea 039c652b28 DOC: Update @dadosfera/protospack-v2 to version 3.38.0-beta.19 in package.json and package-lock.json 2025-12-18 17:11:18 -03:00
viniciusgadea 653d4f53b5 FEAT: Add optional 'docs' field to IDataAsset, IUpdateDataRequest, and ICreateDataAsset 2025-12-18 16:16:00 -03:00
Marcos Rodrigues Silva 31f8c2c1a6 Merge pull request #424 from dadosfera/feature/user-form
FEAT: protected update route with admin seqid or if user request to e…
2025-12-18 12:33:29 -03:00
Marcos Rodrigues 8b93d4e97b FEAT: protected update route with admin seqid or if user request to equal a jwt user id claim 2025-12-18 12:32:53 -03:00
Rafael Santana adeb022818 Merge pull request #423 from dadosfera/fix/reference-column-object-format
FIX: inject customer_id in pipeline execute/pause/unpause routes
2025-12-18 10:53:31 -03:00
Marcos Rodrigues Silva e326cab44d Merge pull request #422 from dadosfera/feature/user-form
FIX: send new user info
2025-12-18 10:29:09 -03:00
Marcos Rodrigues 61109f8ae9 FIX: send new user info 2025-12-18 10:24:29 -03:00
Marcos Rodrigues Silva 3f8dc5cabe Merge pull request #421 from dadosfera/feature/user-form
Feature/user form
2025-12-18 09:53:12 -03:00
Marcos Rodrigues 7f5d157739 FEAT: update protospack version 2025-12-18 09:52:11 -03:00
Marcos Rodrigues bce73fb11f FEAT: update user route 2025-12-17 17:24:03 -03:00
Rafael Santana 9473e65deb Merge pull request #419 from dadosfera/fix/reference-column-object-format
FIX: store reference_column as object with name and type
2025-12-17 13:48:38 -03:00
Rafael Santana 77b9acd2d0 Merge pull request #418 from dadosfera/fix/reference-column-object-format
fix: store reference_column as object with name and type
2025-12-17 13:46:05 -03:00
RafaelandClaude Opus 4.5 e99306adba fix: store reference_column as object with name and type
The protobuf definition expects reference_column to be an object with
name and type fields, but it was being stored as just a string (column
name). This caused pipeline fetching to fail with the error:
".NewTable.reference_column: object expected"

Changes:
- Update ReferenceColumn interface in DynamoDB service
- Update extractTablesFromJobs to create reference_column object
- Update syncJobInputToDynamoDB to handle reference_column object

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 13:45:18 -03:00
Rafael Santana 96b947ebdc Merge pull request #417 from dadosfera/fix/redis-tls-configurable
FIX: pass type field to Elasticsearch createPipeline
2025-12-16 21:10:19 -03:00
Rafael Santana 6f7436f33f Merge pull request #416 from dadosfera/fix/redis-tls-configurable
fix: pass type field to Elasticsearch createPipeline
2025-12-16 21:05:55 -03:00
RafaelandClaude Opus 4.5 865140e681 fix: pass type field to Elasticsearch createPipeline
The type field was missing from the createPipeline call,
causing ES documents to not have the type field set.

Maps: jdbc->database, singer->application, s3->file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 21:05:21 -03:00
Rafael Santana c4a664572a Merge pull request #415 from dadosfera/fix/redis-tls-configurable
UPDATE: force deployment
2025-12-16 19:17:08 -03:00
Rafael Santana 7a10f88113 Merge pull request #414 from dadosfera/fix/redis-tls-configurable
fix: normalize IDs in pipeline run routes before calling Platform API
2025-12-16 19:11:09 -03:00
Rafael Santana 466f8fb8cc Merge pull request #413 from dadosfera/fix/redis-tls-configurable
UPDATE: force deployment
2025-12-16 18:23:44 -03:00
Rafael Santana 5989822263 Merge pull request #412 from dadosfera/fix/redis-tls-configurable
feat: add Elasticsearch access to Maestro deployment
2025-12-16 18:15:08 -03:00
vinicius gadea 64a3e2652e Merge pull request #410 from dadosfera/feature/catalog-update
Feature/catalog update
2025-12-16 18:01:47 -03:00
Rafael Moraes ea44a1cbb6 FEAT: Manual/Auto filter and UX Adjusments 2025-12-16 20:52:28 +00:00
Rafael Moraes fc9c0b0991 WIP: Filtro Source - backend 2025-12-16 20:13:20 +00:00
Rafael Moraes a4b5a44e44 Merge branch 'feature/catalog-update' of github.com:dadosfera/maestro into feature/catalog-update 2025-12-16 20:01:19 +00:00
Rafael Moraes d99a6aa322 Merge branch 'beta' of github.com:dadosfera/maestro into feature/catalog-update 2025-12-16 19:58:48 +00:00
Rafael Santana 3f910f851a Merge pull request #411 from dadosfera/fix/redis-tls-configurable
Fix/redis tls configurable
2025-12-16 16:58:41 -03:00
Rafael Moraes bd231382eb WIP: adiciona parâmetro asset_type no docsfera.json 2025-12-16 19:58:38 +00:00
Rafael Santana 9c1979e17a Merge pull request #405 from dadosfera/fix/redis-tls-configurable
Fix/redis tls configurable
2025-12-16 16:41:04 -03:00
viniciusgadea fcf7fb054e FIX: Update searchDataAssets to handle owner filtering and convert owner names to IDs 2025-12-16 09:55:24 -03:00
viniciusgadea 288aaeabc0 FIX: Enhance filter handling in BuildDataAssetsFilterTerms to support empty owner condition 2025-12-15 12:00:50 -03:00
viniciusgadea 918c3d7416 FIX: Enhance searchDataAssets to convert owner names to IDs and log the conversion 2025-12-15 09:08:08 -03:00
Marcos Rodrigues Silva e7f410831f Merge pull request #395 from dadosfera/beta
Beta
2025-12-01 17:47:24 -03:00
25 changed files with 3983 additions and 3163 deletions
+12
View File
@@ -0,0 +1,12 @@
node_modules
dist
.git
*.log
npm-debug.log*
.DS_Store
.env
.env.*
coverage
.nyc_output
*.tgz
!protospack.tgz
+2 -1
View File
@@ -1,4 +1,5 @@
FROM node:18.17-alpine AS base_image
FROM node:20-alpine AS base_image
RUN npm install -g npm@latest
FROM base_image AS build_base
WORKDIR /app
+47
View File
@@ -0,0 +1,47 @@
FROM node:22-alpine AS base_image
RUN npm install -g npm@latest
FROM base_image AS build_base
WORKDIR /app
RUN apk update
RUN apk add --no-cache \
aws-cli \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
COPY package*.json ./
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# Local build with secrets
FROM build_base AS build
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
RUN npm ci
COPY . .
RUN npm run build
FROM base_image
WORKDIR /app
COPY --from=build /app/dist ./dist
COPY --from=build /app/node_modules ./node_modules
COPY --from=build /app/package*.json ./
RUN apk update
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT ["npm", "run", "start:prod"]
@@ -48,6 +48,9 @@ spec:
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
env:
# Auth Provider Configuration (cognito or keycloak)
- name: AUTH_PROVIDER
value: {{ .Values.maestro.auth_provider | default "cognito" | quote }}
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
- name: AWS_REGION
@@ -108,6 +111,8 @@ spec:
value: "{{ .Values.maestro.redis_tls }}"
- name: PLATFORM_API_URL
value: {{ .Values.maestro.platform_api_url }}
- name: STORAGE_EXPLORER_API_URL
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
+4
View File
@@ -27,6 +27,9 @@ resources:
cpu: 2000m
memory: 2Gi
maestro:
# Auth provider: "cognito" (default) or "keycloak"
# Note: maestro doesn't connect to Keycloak directly, only duc does
auth_provider: "cognito"
aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
duc_url: duc.dadosfera.ai
in_factory_url: in-factory.dadosfera.ai
@@ -44,6 +47,7 @@ maestro:
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api"
dedicated_proxy: ""
restricted_ip: ""
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
+868 -38
View File
File diff suppressed because it is too large Load Diff
+2109 -2969
View File
File diff suppressed because it is too large Load Diff
+10 -4
View File
@@ -30,12 +30,12 @@
"@aws-crypto/sha256-js": "^5.2.0",
"@aws-sdk/client-dynamodb": "^3.414.0",
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/credential-provider-node": "^3.940.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.16",
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -49,7 +49,7 @@
"@nestjs/schematics": "^9.2.0",
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"axios": "^0.30.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
@@ -80,7 +80,13 @@
"swagger-ui-express": "^4.6.3"
},
"overrides": {
"multer": "1.4.5-lts.1"
"multer": "2.0.2",
"form-data": "^4.0.4",
"body-parser": "^1.20.3",
"cross-spawn": "^7.0.5",
"glob": "^10.5.0",
"path-to-regexp": "^3.3.0",
"semver": "^7.5.2"
},
"devDependencies": {
"@types/cache-manager": "^4.0.6",
+2
View File
@@ -34,6 +34,7 @@ import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
@Module({
providers: [
@@ -75,6 +76,7 @@ import { PlatformApiModule } from './modules/platform-api/platform-api.module';
IdentityProviderModule,
NetworkPolicyModule,
PlatformApiModule,
StorageExplorerModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
],
+29
View File
@@ -668,6 +668,35 @@ export const PERMISSIONS_GROUPS = {
},
},
},
STORAGE_EXPLORER: {
title: {
'pt-br': 'Storage Explorer',
'en-us': 'Storage Explorer',
'es-es': 'Storage Explorer',
},
permissions: {
READ: {
seqid: 51,
claim: 'storage-explorer:read',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Ler dados do Storage Explorer',
'en-us': 'Read Storage Explorer data',
'es-es': 'Leer datos del Storage Explorer',
},
},
WRITE: {
seqid: 52,
claim: 'storage-explorer:write',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Escrever dados no Storage Explorer',
'en-us': 'Write Storage Explorer data',
'es-es': 'Escribir datos en Storage Explorer',
},
},
},
},
};
export interface DadosferaModule {
name: string;
+4 -2
View File
@@ -190,13 +190,14 @@ export class AuthController {
) {
this.logger.info('/auth - change-password');
const { oldPassword, newPassword } = body;
const { oldPassword, newPassword, totpCode } = body;
const { authorization: accessToken } = headers;
return this.authClient.changePassword({
accessToken,
oldPassword,
newPassword,
totpCode,
});
}
@@ -214,7 +215,8 @@ export class AuthController {
const { username } = body;
return this.authClient.resetPassword({ username }, metadata);
await this.authClient.resetPassword({ username }, metadata);
return { authProvider: process.env.AUTH_PROVIDER || 'cognito' };
}
@ApiInternalOnlyEndpoint()
+2
View File
@@ -143,6 +143,7 @@ export class AuthClientService implements OnModuleInit {
accessToken,
oldPassword,
newPassword,
totpCode,
}: AuthChangePasswordRequest) {
this.logger.info('ChangePassword');
@@ -151,6 +152,7 @@ export class AuthClientService implements OnModuleInit {
accessToken,
oldPassword,
newPassword,
totpCode,
}),
);
}
+161 -124
View File
@@ -41,7 +41,6 @@ import {
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
catalogWriteService: WriteService.CatalogWriteServices;
@@ -58,7 +57,6 @@ class CatalogService implements OnModuleInit {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
@@ -74,81 +72,65 @@ class CatalogService implements OnModuleInit {
);
}
_getNimbusUrl(body) {
this.logger.debug(`Body: ${JSON.stringify(body)}`);
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
return `https://nimbus-${customer}.dadosfera.ai`;
}
return `https://nimbus-${customer}.${process.env.ENV.replace(
'local',
'stg',
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
this.logger.info('getPiiReporter: ' + type);
try {
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
const { data } = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata),
);
this.logger.info('Finish grpc call');
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
this.logger.info('parser file to: ' + type);
const file = await parser.parse(data);
this.logger.info('finish parser');
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
csv: 'text/csv',
html: 'text/html',
pdf: 'application/pdf',
};
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
type: mimeTypes[type]
}
type: mimeTypes[type],
};
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
return lastValueFrom(
this.catalogWriteService.CreateDataAsset(data, metadata),
);
}
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.ManagePermission(data, metadata),
).catch((err) => {
@@ -159,11 +141,9 @@ class CatalogService implements OnModuleInit {
});
}
async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.RevokePermission(data, metadata),
).catch((err) => {
@@ -174,11 +154,9 @@ class CatalogService implements OnModuleInit {
});
}
async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.MakeAComment(data, metadata),
).catch((err) => {
@@ -189,11 +167,9 @@ class CatalogService implements OnModuleInit {
});
}
async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.UpdateAComment(data, metadata),
).catch((err) => {
@@ -204,11 +180,9 @@ class CatalogService implements OnModuleInit {
});
}
async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
return lastValueFrom(
this.catalogWriteService.DeleteDataAsset(data, metadata),
).catch((err) => {
@@ -219,30 +193,114 @@ class CatalogService implements OnModuleInit {
});
}
async getUserRolesIds(userId: string) {
const result = await this.userService.findOneById(userId).catch(() => null);
if (result) {
return result.user.roles.map((role) => role.id);
}
return [];
}
async searchDataAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
this.logger.info('CatalogService - searchDataAssets');
this.logger.info('CatalogService - searchDataAssets', { query });
const { search, page, size, sort_by, order, ...filters } = query;
this.logger.debug('Extracted filters:', { filters });
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (
filters.manually !== undefined &&
filters.manually !== null &&
filters.manually !== ''
) {
filters.manually = Number(filters.manually); // 1 ou 0
} else {
delete filters.manually;
}
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (filters.owner) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
this.logger.info('Available users in database count:', {
count: customer_users.length,
});
this.logger.info('First 5 users:', {
users: customer_users
.slice(0, 5)
.map((u) => ({ id: u.id, email: u.email, name: u.name })),
});
const ownerValues = Array.isArray(filters.owner)
? filters.owner
: typeof filters.owner === 'string' && filters.owner.includes(',')
? filters.owner.split(',').map((o: string) => o.trim())
: [filters.owner];
this.logger.info('Owner values to convert:', {
ownerValues,
ownerFiltersOriginal: filters.owner,
});
const ownerIds = ownerValues
.map((ownerValue: string) => {
const normalizedOwner = ownerValue.replace(/\s/g, '+');
const user = customer_users.find((u) => {
const isIdMatch = u.id === ownerValue;
const isEmailMatch =
u.email === ownerValue || u.email === normalizedOwner;
const isNameMatch =
u.name === ownerValue || u.name === normalizedOwner;
this.logger.info('Comparing:', {
userId: u.id,
userEmail: u.email,
userName: u.name,
filterValue: ownerValue,
normalizedFilter: normalizedOwner,
idMatch: isIdMatch,
emailMatch: isEmailMatch,
nameMatch: isNameMatch,
});
return isIdMatch || isEmailMatch || isNameMatch;
});
this.logger.info('Looking for owner result:', {
ownerValue,
found: !!user,
userId: user?.id,
});
return user?.id || ownerValue;
})
.filter((id: string) => id);
if (ownerIds.length > 0) {
filters.owner = ownerIds;
}
}
const { data_assets, total } = await lastValueFrom(
this.catalogReadService.GetAllDataAssets(
{
@@ -257,20 +315,18 @@ class CatalogService implements OnModuleInit {
),
);
console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
const result = JSON.parse(data_assets);
const response = await this.getAssetsUsersAndRoles(
result.data_assets,
customer_id,
);
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
@@ -278,33 +334,27 @@ class CatalogService implements OnModuleInit {
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
const formatData = data.data_assets.map((asset) => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
tags: '[' + asset.tags.join(', ') + ']',
}));
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
filename,
};
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -325,11 +375,9 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getOneDataAssetByPipelineAndObject(data: {
customer_id: string;
pipeline: string;
@@ -351,11 +399,9 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
@@ -364,7 +410,6 @@ class CatalogService implements OnModuleInit {
}) {
const { body, customer_id, data_asset_id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{ id: data_asset_id, changes: JSON.stringify(body) },
@@ -379,7 +424,6 @@ class CatalogService implements OnModuleInit {
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
@@ -387,12 +431,11 @@ class CatalogService implements OnModuleInit {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id }, metadata),
);
const docs = JSON.parse(documentation);
return docs;
}
async getDatasetPreview(id: string, metadata: Metadata) {
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
@@ -404,7 +447,6 @@ class CatalogService implements OnModuleInit {
return result;
}
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
@@ -420,7 +462,7 @@ class CatalogService implements OnModuleInit {
if (body.asset_type === 'table' || body.asset_type === 'view') {
return this.createDataDocsViaNimbus(body);
}
return this.createDataDocsViaGrpc(body, metadata);
}
@@ -434,7 +476,6 @@ class CatalogService implements OnModuleInit {
return data;
}
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
this.logger.info('Creating data docs via gRPC for other asset types');
try {
@@ -447,9 +488,8 @@ class CatalogService implements OnModuleInit {
metadata,
),
);
return response;
} catch (error) {
this.logger.error('Error creating data asset docs:', error);
throw new HttpException(
@@ -462,7 +502,6 @@ class CatalogService implements OnModuleInit {
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
const response = await lastValueFrom(
this.catalogReadService.GetCustomerTags(data, metadata),
)
@@ -475,9 +514,9 @@ class CatalogService implements OnModuleInit {
throw new Error(err);
});
return response;
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
@@ -490,17 +529,15 @@ class CatalogService implements OnModuleInit {
(u) => u.id === data_asset.owner,
)?.email;
const roles = [];
const users = [];
const data_asset_roles = data_asset?.roles || []
const data_asset_roles = data_asset?.roles || [];
for (const role_id of data_asset_roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
const data_asset_users = data_asset?.users || []
const data_asset_users = data_asset?.users || [];
for (const user_id of data_asset_users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, email: user.email });
@@ -514,7 +551,6 @@ class CatalogService implements OnModuleInit {
});
}
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
const { session } = await lastValueFrom(
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -528,7 +564,6 @@ class CatalogService implements OnModuleInit {
return res;
}
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.AddRlsRule(data, metadata),
@@ -536,7 +571,6 @@ class CatalogService implements OnModuleInit {
return res;
}
async removeRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
@@ -544,14 +578,12 @@ class CatalogService implements OnModuleInit {
return res;
}
async batchRemoveRlsRule(
query: BatchRemoveRlsRulesRequest,
metadata: Metadata,
) {
const { id_rls, nimbus_dashboard_id } = query;
if (id_rls && nimbus_dashboard_id) {
throw new BadRequestException(
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
@@ -572,7 +604,6 @@ class CatalogService implements OnModuleInit {
return 'OK';
}
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetRlsRules(data, metadata),
@@ -580,7 +611,6 @@ class CatalogService implements OnModuleInit {
return res.rls_rules;
}
async getOneRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetOneRlsRule({ id }, metadata),
@@ -588,7 +618,6 @@ class CatalogService implements OnModuleInit {
return res.rls_rule;
}
async getNimbusDashboards(
data: GetNimbusDashboardsRequest,
metadata: Metadata,
@@ -599,88 +628,99 @@ class CatalogService implements OnModuleInit {
return res.dashboards;
}
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
this.logger.info(
`Creating table metadata for table ${body.table_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {
...body.logMetadata,
});
try {
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
const { data, status } = await axios.post(endpoint, {
...body.table_metadata,
});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
} because of ${JSON.stringify(error.response?.data) || error.message}`,
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
async createColumnMetadata(body: any): Promise<number[]> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Creating column metadata for table ${body.column_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.column_metadata)}`,
{ ...body.logMetadata },
);
const { data, status } = await axios.post(
endpoint,
body.column_metadata,
);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
error.response?.status
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
} because of ${error.response?.data || error.message}`,
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
this.logger.info(
`Creating data preview for table ${body.data_preview.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.data_preview)}`,
{ ...body.logMetadata },
);
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
return data.id;
} catch (error) {
@@ -688,17 +728,15 @@ class CatalogService implements OnModuleInit {
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{...body.logMetadata},
{ ...body.logMetadata },
);
throw new Error(error.response?.data?.message || error.message);
}
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
const customer_name = customer_name_raw?.[0]?.toString();
if (!customer_name) {
throw new BadRequestException('Customer name not found in metadata');
@@ -721,5 +759,4 @@ class CatalogService implements OnModuleInit {
}
}
export { CatalogService };
+6
View File
@@ -98,6 +98,8 @@ export class IDataAsset {
embed?: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class IOneDataAsset {
@@ -200,6 +202,8 @@ export class IUpdateDataRequest {
embed: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
@@ -214,6 +218,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
location: string;
@ApiPropertyOptional()
embed: EmbedObject;
@ApiPropertyOptional()
docs: string;
}
export class IPreview {
@@ -136,4 +136,32 @@ export class CustomersController {
const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
return result;
}
@Get(':id/organization-info')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ description: 'Organization information' })
async getOrganizationInfo(@Param('id') id: string) {
this.logger.info('getOrganizationInfo', { id });
return this.customersService.getOrganizationInfo(id);
}
@Put(':id/organization-info')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ description: 'Organization information updated' })
async updateOrganizationInfo(
@Param('id') id: string,
@Body() body: {
companyName: string;
companySite: string;
domain: string;
cnpj: string;
description: string;
},
) {
return this.customersService.updateOrganizationInfo(id, body);
}
}
+54 -1
View File
@@ -223,4 +223,57 @@ export class CustomersService implements OnModuleInit {
})
)
}
}
async updateOrganizationInfo(
customerId: string,
data: {
companyName: string;
companySite: string;
domain: string;
cnpj: string;
description: string;
},
) {
try {
const result = await lastValueFrom(
this.customerService.OrganizationUpdate({
customerId,
companyName: data.companyName || '',
companySite: data.companySite || '',
domain: data.domain || '',
cnpj: data.cnpj || '',
description: data.description || '',
}),
);
return result;
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
async getOrganizationInfo(customerId: string) {
try {
const customerResponse = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId })
);
const customer = customerResponse.customer;
return {
companyName: customer.companyName || '',
companySite: customer.companySite || '',
domain: customer.domain || '',
cnpj: customer.cnpj || '',
description: customer.description || ''
};
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
}
@@ -0,0 +1,27 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class OrganizationUpdateRequest {
@ApiProperty()
name: string;
@ApiPropertyOptional()
companySite: string;
@ApiProperty()
domain: string;
@ApiPropertyOptional()
info: string;
@ApiPropertyOptional()
cnpj: string;
}
export class OrganizationResponse {
@ApiProperty()
name: string;
@ApiPropertyOptional()
companySite: string;
@ApiProperty()
domain: string;
@ApiPropertyOptional()
info: string;
@ApiPropertyOptional()
cnpj: string;
}
+22 -16
View File
@@ -229,27 +229,33 @@ export class RolesService {
const [roleTreated] = this.getRolesPermissionsName([role.role]);
return { role: roleTreated };
}
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
const newRoles: RoleDto[] = [];
for (const role of roles) {
const allPermissions = this.permissionsService.getAllPermissions(
this.language,
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
const newRole: RoleDto = {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};
const newRole: RoleDto = this.formatRole(role);
newRoles.push(newRole);
}
return newRoles;
}
private formatRole(role: GetRolesPermissionsName): RoleDto {
const allPermissions = this.permissionsService.getAllPermissions(
this.language
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
return {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};;
}
}
@@ -0,0 +1,13 @@
export const STORAGE_EXPLORER_CONFIG = {
getUrl: (customerId: string): string => {
const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
if (!urlTemplate) {
throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
}
// Replace {customer_id} placeholder with actual customer ID
// For local: http://172.17.0.1:8000/api (no placeholder)
// For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api
return urlTemplate.replace('{customer_id}', customerId);
},
timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
};
@@ -0,0 +1,350 @@
import {
Controller,
Get,
Post,
Put,
Delete,
Param,
Body,
Query,
Inject,
UseInterceptors,
UploadedFiles,
} from '@nestjs/common';
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
import { FilesInterceptor } from '@nestjs/platform-express';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import FormData from 'form-data';
import {
Authenticated,
RequireAllPermissions,
} from '../../decorators/authentication.decorator';
import { User, RequestUser } from '../../decorators/user.decorator';
import { StorageExplorerService } from './storage-explorer.service';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
@ApiTags('Storage Explorer')
@Controller('storage-explorer')
@Authenticated()
export class StorageExplorerController {
private logger: any;
constructor(
private readonly storageExplorerService: StorageExplorerService,
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
// ============================================
// TABLE OPERATIONS
// ============================================
@ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' })
@Post('tables/validate-name')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async validateTableName(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/validate-name',
user,
body,
);
}
@ApiOperation({ summary: 'Create a new table' })
@Post('tables')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async createTable(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/',
user,
body,
);
}
@ApiOperation({ summary: 'List all tables with pagination' })
@Get('tables')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listTables(
@Query('page') page: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/tables/',
user,
undefined,
{ page },
);
}
@ApiOperation({ summary: 'Get table details by ID' })
@Get('tables/:tableId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTable(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}`,
user,
);
}
@ApiOperation({ summary: 'Link a dataset to a table' })
@Post('tables/:tableId/datasets/:datasetId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async linkDatasetToTable(
@Param('tableId') tableId: string,
@Param('datasetId') datasetId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
`/tables/${tableId}/datasets/${datasetId}`,
user,
);
}
@ApiOperation({ summary: 'Get all datasets linked to a table' })
@Get('tables/:tableId/datasets')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTableDatasets(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/datasets`,
user,
);
}
@ApiOperation({ summary: 'Get table schema' })
@Get('tables/:tableId/schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTableSchema(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/schema`,
user,
);
}
@ApiOperation({ summary: 'Validate schema compatibility between table and dataset' })
@Post('tables/:tableId/validate-compatibility/:datasetId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async validateSchemaCompatibility(
@Param('tableId') tableId: string,
@Param('datasetId') datasetId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
`/tables/${tableId}/validate-compatibility/${datasetId}`,
user,
);
}
// ============================================
// DATASET OPERATIONS
// ============================================
@ApiOperation({ summary: 'Get dataset preview data' })
@Get('datasets/:datasetId/preview')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getDatasetPreview(
@Param('datasetId') datasetId: string,
@Query('limit') limit: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/preview`,
user,
undefined,
{ limit },
);
}
@ApiOperation({ summary: 'Get dataset schema information' })
@Get('datasets/:datasetId/schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getDatasetSchema(
@Param('datasetId') datasetId: string,
@Query('force_refresh') forceRefresh: boolean,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/schema`,
user,
undefined,
{ force_refresh: forceRefresh },
);
}
@ApiOperation({ summary: 'List all datasets for a specific upload' })
@Get('datasets/upload/:uploadId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listDatasetsByUpload(
@Param('uploadId') uploadId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/upload/${uploadId}`,
user,
);
}
@ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' })
@Put('datasets/:datasetId/refresh-schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async refreshDatasetSchema(
@Param('datasetId') datasetId: string,
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'PUT',
`/datasets/${datasetId}/refresh-schema`,
user,
body,
);
}
// ============================================
// STORAGE OPERATIONS
// ============================================
@ApiOperation({ summary: 'List file explorer uploads with pagination' })
@Get('storage/uploads/history')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listFileExplorerUploads(
@Query('page') page: number,
@Query('limit') limit: number,
@Query('folder_path') folderPath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/uploads/history',
user,
undefined,
{ page, limit, folder_path: folderPath },
);
}
@ApiOperation({ summary: 'Browse folders and files in storage' })
@Get('storage/browse')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async browseStorage(
@Query('path') path: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/browse',
user,
undefined,
{ path },
);
}
@ApiOperation({ summary: 'Upload multiple files to storage' })
@Post('storage/upload/batch')
@ApiConsumes('multipart/form-data')
@UseInterceptors(FilesInterceptor('files'))
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async batchUpload(
@UploadedFiles() files: Array<Express.Multer.File>,
@Body('folder_path') folderPath: string,
@User() user: RequestUser,
) {
// Create FormData to forward files to storage-explorer API
const formData = new FormData();
// Add files
if (files && files.length > 0) {
files.forEach((file) => {
formData.append('files', file.buffer, {
filename: file.originalname,
contentType: file.mimetype,
});
});
}
// Add folder_path
if (folderPath) {
formData.append('folder_path', folderPath);
}
return this.storageExplorerService.proxyFormData(
'POST',
'/storage/upload/batch',
user,
formData,
);
}
@ApiOperation({ summary: 'Create a new folder in storage' })
@Post('storage/folder/create')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async createFolder(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/storage/folder/create',
user,
body,
);
}
@ApiOperation({ summary: 'Download a file from storage' })
@Get('storage/download')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async downloadFile(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/download',
user,
undefined,
{ file_path: filePath },
);
}
@ApiOperation({ summary: 'Get detailed file metadata' })
@Get('storage/metadata')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getFileMetadata(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/metadata',
user,
undefined,
{ file_path: filePath },
);
}
}
@@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { StorageExplorerController } from './storage-explorer.controller';
import { StorageExplorerService } from './storage-explorer.service';
@Module({
imports: [],
controllers: [StorageExplorerController],
providers: [StorageExplorerService, DadosferaLogger],
exports: [StorageExplorerService],
})
export class StorageExplorerModule {}
@@ -0,0 +1,178 @@
import { Injectable, Inject, HttpException } from '@nestjs/common';
import axios, { AxiosResponse, Method } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser } from '../../decorators/user.decorator';
import { STORAGE_EXPLORER_CONFIG } from './storage-explorer.config';
@Injectable()
export class StorageExplorerService {
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async proxy(
method: string,
path: string,
user: RequestUser,
body?: any,
query?: Record<string, any>,
): Promise<any> {
// Validate customer_id is present for multi-tenant isolation
if (!user.customer_id) {
throw new HttpException('Customer ID is required for storage operations', 400);
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
'content-type': 'application/json',
};
this.logger.info('Proxying request to storage-explorer', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
storage_url: baseUrl,
user_id: user.user_id,
});
try {
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers,
data: body,
timeout: STORAGE_EXPLORER_CONFIG.timeout,
validateStatus: () => true, // Don't throw on non-2xx
});
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Storage Explorer API proxy error', {
error: error.message,
status: error.response?.status,
path,
storage_url: baseUrl,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
if (error.code === 'ECONNREFUSED') {
throw new HttpException('Storage Explorer API service unavailable', 503);
}
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
throw new HttpException('Storage Explorer API request timeout', 504);
}
throw new HttpException('Internal server error', 500);
}
}
/**
* Proxy with file upload support (multipart/form-data)
*/
async proxyFormData(
method: string,
path: string,
user: RequestUser,
formData: any,
query?: Record<string, any>,
): Promise<any> {
// Validate customer_id is present for multi-tenant isolation
if (!user.customer_id) {
throw new HttpException('Customer ID is required for storage operations', 400);
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
// Let axios set Content-Type for multipart/form-data with boundary
...formData.getHeaders?.(),
};
this.logger.info('Proxying form data request to storage-explorer', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
storage_url: baseUrl,
user_id: user.user_id,
});
try {
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers,
data: formData,
timeout: STORAGE_EXPLORER_CONFIG.timeout,
maxContentLength: Infinity,
maxBodyLength: Infinity,
validateStatus: () => true,
});
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Storage Explorer API form data proxy error', {
error: error.message,
status: error.response?.status,
path,
storage_url: baseUrl,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
throw new HttpException('Internal server error', 500);
}
}
}
+14
View File
@@ -38,6 +38,12 @@ export class User {
department?: string;
@ApiProperty()
hierarchy?: string;
@ApiProperty()
bio?: string;
@ApiProperty()
companyName?: string;
@ApiProperty()
personalSite?: string;
@ApiPropertyOptional()
customer?: Customer;
@ApiProperty()
@@ -58,6 +64,8 @@ export class UserNoRolesAndCustomer extends OmitType(UserNoRoles, [
export class IUserByCustomer extends OmitType(User, ['customer']) {
@ApiPropertyOptional()
permissions?: string[];
@ApiPropertyOptional()
authProvider?: string;
}
export class CreateUserReq {
@@ -109,6 +117,12 @@ export class UpdateUserReq {
@ApiPropertyOptional()
hierarchy?: string;
@ApiPropertyOptional()
bio?: string;
@ApiPropertyOptional()
personalSite?: string;
@ApiPropertyOptional()
companyName?: string;
@ApiPropertyOptional()
roleNames?: string[];
}
+12 -1
View File
@@ -266,7 +266,6 @@ export class UsersController {
}
@Patch(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: UpdateUserRes })
async updateUser(
@User() user: RequestUser,
@@ -274,6 +273,18 @@ export class UsersController {
@Param('id') id: string,
@Language() language: LanguageEnum,
) {
const isSameUser = user.user_id === id;
const isSuperAdmin = user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)
if (!isSameUser && !isSuperAdmin) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
if (isSameUser && !isSuperAdmin && body.roleNames) {
// Prevent users from updating their own roles
delete body.roleNames;
}
this.logger.info('updateUser', { user });
this.userService.setLanguage(language);
return await this.userService.updateUser(body, id, user.customer_id);
+11 -7
View File
@@ -125,6 +125,7 @@ export class UsersService implements OnModuleInit {
return { permissions };
});
res.user.permissions = permissions;
res.user.authProvider = process.env.AUTH_PROVIDER || 'cognito';
return res;
}
@@ -148,20 +149,23 @@ export class UsersService implements OnModuleInit {
}
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
if (roleNames) {
const { roleNames, ...updateUserDTO } = req;
if (roleNames && roleNames.length > 0) {
await this.setRoles({ roleNames, userId: id }, customerId);
}
const { user } = await lastValueFrom(
this.usersClientService.UserUpdate({
name,
department: updateUserDTO.department,
email: updateUserDTO.email,
hierarchy: updateUserDTO.hierarchy,
jobTitle: updateUserDTO.jobTitle,
name: updateUserDTO.name,
bio: updateUserDTO.bio,
companyName: updateUserDTO.companyName,
personalSite: updateUserDTO.personalSite,
customerId,
id,
department,
hierarchy,
jobTitle,
email,
metabaseUserId: undefined,
}),
);