From a2c7ce00db92d11eb674d62d82006f64a937a280 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 13 Jan 2026 10:29:38 -0300 Subject: [PATCH 1/6] FEAT: integrate Storage Explorer API with new endpoints and permissions --- docsfera.json | 751 ++++++++++++++++++ package-lock.json | 1 - src/app.module.ts | 2 + src/authentication/permissions.enum.ts | 29 + .../storage-explorer.config.ts | 13 + .../storage-explorer.controller.ts | 384 +++++++++ .../storage-explorer.module.ts | 13 + .../storage-explorer.service.ts | 190 +++++ 8 files changed, 1382 insertions(+), 1 deletion(-) create mode 100644 src/modules/storage-explorer/storage-explorer.config.ts create mode 100644 src/modules/storage-explorer/storage-explorer.controller.ts create mode 100644 src/modules/storage-explorer/storage-explorer.module.ts create mode 100644 src/modules/storage-explorer/storage-explorer.service.ts diff --git a/docsfera.json b/docsfera.json index 8942a26..92b3f98 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7767,6 +7767,757 @@ ] } }, + "/storage-explorer/tables/validate-name": { + "post": { + "operationId": "StorageExplorerController_validateTableName", + "summary": "Validate table name in PostgreSQL and Snowflake", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables": { + "post": { + "operationId": "StorageExplorerController_createTable", + "summary": "Create a new table", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "StorageExplorerController_listTables", + "summary": "List all tables with pagination", + "parameters": [ + { + "name": "page", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}": { + "get": { + "operationId": "StorageExplorerController_getTable", + "summary": "Get table details by ID", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/datasets/{datasetId}": { + "post": { + "operationId": "StorageExplorerController_linkDatasetToTable", + "summary": "Link a dataset to a table", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/datasets": { + "get": { + "operationId": "StorageExplorerController_getTableDatasets", + "summary": "Get all datasets linked to a table", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/schema": { + "get": { + "operationId": "StorageExplorerController_getTableSchema", + "summary": "Get table schema", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/validate-compatibility/{datasetId}": { + "post": { + "operationId": "StorageExplorerController_validateSchemaCompatibility", + "summary": "Validate schema compatibility between table and dataset", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/{datasetId}/preview": { + "get": { + "operationId": "StorageExplorerController_getDatasetPreview", + "summary": "Get dataset preview data", + "parameters": [ + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/{datasetId}/schema": { + "get": { + "operationId": "StorageExplorerController_getDatasetSchema", + "summary": "Get dataset schema information", + "parameters": [ + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "force_refresh", + "required": true, + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/upload/{uploadId}": { + "get": { + "operationId": "StorageExplorerController_listDatasetsByUpload", + "summary": "List all datasets for a specific upload", + "parameters": [ + { + "name": "uploadId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/{datasetId}/refresh-schema": { + "put": { + "operationId": "StorageExplorerController_refreshDatasetSchema", + "summary": "Refresh dataset schema with new parsing options (Excel)", + "parameters": [ + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/uploads/history": { + "get": { + "operationId": "StorageExplorerController_listFileExplorerUploads", + "summary": "List file explorer uploads with pagination", + "parameters": [ + { + "name": "page", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + }, + { + "name": "limit", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + }, + { + "name": "folder_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/browse": { + "get": { + "operationId": "StorageExplorerController_browseStorage", + "summary": "Browse folders and files in storage", + "parameters": [ + { + "name": "path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/upload/batch": { + "post": { + "operationId": "StorageExplorerController_batchUpload", + "summary": "Upload multiple files to storage", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/folder/create": { + "post": { + "operationId": "StorageExplorerController_createFolder", + "summary": "Create a new folder in storage", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/download": { + "get": { + "operationId": "StorageExplorerController_downloadFile", + "summary": "Download a file from storage", + "parameters": [ + { + "name": "file_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/delete": { + "delete": { + "operationId": "StorageExplorerController_deleteFile", + "summary": "Delete a file from storage", + "parameters": [ + { + "name": "file_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/metadata": { + "get": { + "operationId": "StorageExplorerController_getFileMetadata", + "summary": "Get detailed file metadata", + "parameters": [ + { + "name": "file_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/health": { "get": { "operationId": "HealthController_check", diff --git a/package-lock.json b/package-lock.json index c27510d..4d90eb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1747,7 +1747,6 @@ "version": "3.38.0-beta.26", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz", "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==", - "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/src/app.module.ts b/src/app.module.ts index 27d042c..0865ad2 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -34,6 +34,7 @@ import { AssignModule } from './modules/assign/assign.module'; import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module'; import { ApiKeyModule } from './modules/api-key/api-key.module'; import { PlatformApiModule } from './modules/platform-api/platform-api.module'; +import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module'; @Module({ providers: [ @@ -75,6 +76,7 @@ import { PlatformApiModule } from './modules/platform-api/platform-api.module'; IdentityProviderModule, NetworkPolicyModule, PlatformApiModule, + StorageExplorerModule, //Always leave HealthModule last, so it is on the bottom of swagger HealthModule, ], diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 3eeb2a7..ca5e9ac 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -668,6 +668,35 @@ export const PERMISSIONS_GROUPS = { }, }, }, + STORAGE_EXPLORER: { + title: { + 'pt-br': 'Storage Explorer', + 'en-us': 'Storage Explorer', + 'es-es': 'Storage Explorer', + }, + permissions: { + READ: { + seqid: 51, + claim: 'storage-explorer:read', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Ler dados do Storage Explorer', + 'en-us': 'Read Storage Explorer data', + 'es-es': 'Leer datos del Storage Explorer', + }, + }, + WRITE: { + seqid: 52, + claim: 'storage-explorer:write', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Escrever dados no Storage Explorer', + 'en-us': 'Write Storage Explorer data', + 'es-es': 'Escribir datos en Storage Explorer', + }, + }, + }, + }, }; export interface DadosferaModule { name: string; diff --git a/src/modules/storage-explorer/storage-explorer.config.ts b/src/modules/storage-explorer/storage-explorer.config.ts new file mode 100644 index 0000000..417eb15 --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.config.ts @@ -0,0 +1,13 @@ +export const STORAGE_EXPLORER_CONFIG = { + getUrl: (customerId: string): string => { + const urlTemplate = process.env.STORAGE_EXPLORER_API_URL; + if (!urlTemplate) { + throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set'); + } + // Replace {customer_id} placeholder with actual customer ID + // For local: http://172.17.0.1:8000/api (no placeholder) + // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api + return urlTemplate.replace('{customer_id}', customerId); + }, + timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10), +}; diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts new file mode 100644 index 0000000..a18f4a9 --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -0,0 +1,384 @@ +import { + Controller, + Get, + Post, + Put, + Delete, + Param, + Body, + Query, + Inject, + UseInterceptors, + UploadedFiles, +} from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger'; +import { FilesInterceptor } from '@nestjs/platform-express'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import FormData from 'form-data'; + +import { + Authenticated, + RequireAllPermissions, +} from '../../decorators/authentication.decorator'; +import { User, RequestUser } from '../../decorators/user.decorator'; +import { StorageExplorerService } from './storage-explorer.service'; +import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; + +@ApiTags('Storage Explorer') +@Controller('storage-explorer') +export class StorageExplorerController { + private logger: any; + + constructor( + private readonly storageExplorerService: StorageExplorerService, + @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + // ============================================ + // TABLE OPERATIONS + // ============================================ + + @ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' }) + @Post('tables/validate-name') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async validateTableName( + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + '/tables/validate-name', + user, + body, + ); + } + + @ApiOperation({ summary: 'Create a new table' }) + @Post('tables') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async createTable( + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + '/tables/', + user, + body, + ); + } + + @ApiOperation({ summary: 'List all tables with pagination' }) + @Get('tables') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async listTables( + @Query('page') page: number, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/tables/', + user, + undefined, + { page }, + ); + } + + @ApiOperation({ summary: 'Get table details by ID' }) + @Get('tables/:tableId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getTable( + @Param('tableId') tableId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/tables/${tableId}`, + user, + ); + } + + @ApiOperation({ summary: 'Link a dataset to a table' }) + @Post('tables/:tableId/datasets/:datasetId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async linkDatasetToTable( + @Param('tableId') tableId: string, + @Param('datasetId') datasetId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + `/tables/${tableId}/datasets/${datasetId}`, + user, + ); + } + + @ApiOperation({ summary: 'Get all datasets linked to a table' }) + @Get('tables/:tableId/datasets') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getTableDatasets( + @Param('tableId') tableId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/tables/${tableId}/datasets`, + user, + ); + } + + @ApiOperation({ summary: 'Get table schema' }) + @Get('tables/:tableId/schema') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getTableSchema( + @Param('tableId') tableId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/tables/${tableId}/schema`, + user, + ); + } + + @ApiOperation({ summary: 'Validate schema compatibility between table and dataset' }) + @Post('tables/:tableId/validate-compatibility/:datasetId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async validateSchemaCompatibility( + @Param('tableId') tableId: string, + @Param('datasetId') datasetId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + `/tables/${tableId}/validate-compatibility/${datasetId}`, + user, + ); + } + + // ============================================ + // DATASET OPERATIONS + // ============================================ + + @ApiOperation({ summary: 'Get dataset preview data' }) + @Get('datasets/:datasetId/preview') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getDatasetPreview( + @Param('datasetId') datasetId: string, + @Query('limit') limit: number, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/datasets/${datasetId}/preview`, + user, + undefined, + { limit }, + ); + } + + @ApiOperation({ summary: 'Get dataset schema information' }) + @Get('datasets/:datasetId/schema') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getDatasetSchema( + @Param('datasetId') datasetId: string, + @Query('force_refresh') forceRefresh: boolean, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/datasets/${datasetId}/schema`, + user, + undefined, + { force_refresh: forceRefresh }, + ); + } + + @ApiOperation({ summary: 'List all datasets for a specific upload' }) + @Get('datasets/upload/:uploadId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async listDatasetsByUpload( + @Param('uploadId') uploadId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/datasets/upload/${uploadId}`, + user, + ); + } + + @ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' }) + @Put('datasets/:datasetId/refresh-schema') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async refreshDatasetSchema( + @Param('datasetId') datasetId: string, + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'PUT', + `/datasets/${datasetId}/refresh-schema`, + user, + body, + ); + } + + // ============================================ + // STORAGE OPERATIONS + // ============================================ + + @ApiOperation({ summary: 'List file explorer uploads with pagination' }) + @Get('storage/uploads/history') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async listFileExplorerUploads( + @Query('page') page: number, + @Query('limit') limit: number, + @Query('folder_path') folderPath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/uploads/history', + user, + undefined, + { page, limit, folder_path: folderPath }, + ); + } + + @ApiOperation({ summary: 'Browse folders and files in storage' }) + @Get('storage/browse') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async browseStorage( + @Query('path') path: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/browse', + user, + undefined, + { path }, + ); + } + + @ApiOperation({ summary: 'Upload multiple files to storage' }) + @Post('storage/upload/batch') + @ApiConsumes('multipart/form-data') + @UseInterceptors(FilesInterceptor('files')) + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async batchUpload( + @UploadedFiles() files: Array, + @Body('folder_path') folderPath: string, + @User() user: RequestUser, + ) { + // Create FormData to forward files to storage-explorer API + const formData = new FormData(); + + // Add files + if (files && files.length > 0) { + files.forEach((file) => { + formData.append('files', file.buffer, { + filename: file.originalname, + contentType: file.mimetype, + }); + }); + } + + // Add folder_path + if (folderPath) { + formData.append('folder_path', folderPath); + } + + return this.storageExplorerService.proxyFormData( + 'POST', + '/storage/upload/batch', + user, + formData, + ); + } + + @ApiOperation({ summary: 'Create a new folder in storage' }) + @Post('storage/folder/create') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async createFolder( + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + '/storage/folder/create', + user, + body, + ); + } + + @ApiOperation({ summary: 'Download a file from storage' }) + @Get('storage/download') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async downloadFile( + @Query('file_path') filePath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/download', + user, + undefined, + { file_path: filePath }, + ); + } + + @ApiOperation({ summary: 'Delete a file from storage' }) + @Delete('storage/delete') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async deleteFile( + @Query('file_path') filePath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'DELETE', + '/storage/delete', + user, + undefined, + { file_path: filePath }, + ); + } + + @ApiOperation({ summary: 'Get detailed file metadata' }) + @Get('storage/metadata') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getFileMetadata( + @Query('file_path') filePath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/metadata', + user, + undefined, + { file_path: filePath }, + ); + } +} diff --git a/src/modules/storage-explorer/storage-explorer.module.ts b/src/modules/storage-explorer/storage-explorer.module.ts new file mode 100644 index 0000000..b0f0e86 --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { StorageExplorerController } from './storage-explorer.controller'; +import { StorageExplorerService } from './storage-explorer.service'; + +@Module({ + imports: [], + controllers: [StorageExplorerController], + providers: [StorageExplorerService, DadosferaLogger], + exports: [StorageExplorerService], +}) +export class StorageExplorerModule {} diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts new file mode 100644 index 0000000..1430dbb --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -0,0 +1,190 @@ +import { Injectable, Inject, HttpException } from '@nestjs/common'; +import axios, { AxiosResponse, Method } from 'axios'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { RequestUser } from '../../decorators/user.decorator'; +import { STORAGE_EXPLORER_CONFIG } from './storage-explorer.config'; + +@Injectable() +export class StorageExplorerService { + private logger: any; + + constructor( + @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + async proxy( + method: string, + path: string, + user: RequestUser, + body?: any, + query?: Record, + ): Promise { + // Validate customer_id is present for multi-tenant isolation + if (!user.customer_id) { + throw new HttpException('Customer ID is required for storage operations', 400); + } + + // Get customer-specific storage-explorer URL + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const url = new URL(`${baseUrl}${path}`); + + // Add query params + if (query) { + Object.entries(query).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + }); + } + + const headers: Record = { + 'content-type': 'application/json', + // Forward user context for logging + 'x-customer-id': user.customer_id, + 'x-customer-name': user.customer_name || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + }; + + this.logger.info('Proxying request to storage-explorer', { + method: method.toUpperCase(), + path, + customer_id: user.customer_id, + storage_url: baseUrl, + user_id: user.user_id, + }); + + try { + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers, + data: body, + timeout: STORAGE_EXPLORER_CONFIG.timeout, + validateStatus: () => true, // Don't throw on non-2xx + }); + + // Propagate non-2xx responses as HttpExceptions + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + + return response.data; + } catch (error) { + this.logger.error('Storage Explorer API proxy error', { + error: error.message, + status: error.response?.status, + path, + storage_url: baseUrl, + method: method.toUpperCase(), + }); + + if (error instanceof HttpException) { + throw error; + } + + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + + if (error.code === 'ECONNREFUSED') { + throw new HttpException('Storage Explorer API service unavailable', 503); + } + + if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') { + throw new HttpException('Storage Explorer API request timeout', 504); + } + + throw new HttpException('Internal server error', 500); + } + } + + /** + * Proxy with file upload support (multipart/form-data) + */ + async proxyFormData( + method: string, + path: string, + user: RequestUser, + formData: any, + query?: Record, + ): Promise { + // Validate customer_id is present for multi-tenant isolation + if (!user.customer_id) { + throw new HttpException('Customer ID is required for storage operations', 400); + } + + // Get customer-specific storage-explorer URL + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const url = new URL(`${baseUrl}${path}`); + + // Add query params + if (query) { + Object.entries(query).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + }); + } + + const headers: Record = { + // Forward user context for logging + 'x-customer-id': user.customer_id, + 'x-customer-name': user.customer_name || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + // Let axios set Content-Type for multipart/form-data with boundary + ...formData.getHeaders?.(), + }; + + this.logger.info('Proxying form data request to storage-explorer', { + method: method.toUpperCase(), + path, + customer_id: user.customer_id, + storage_url: baseUrl, + user_id: user.user_id, + }); + + try { + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers, + data: formData, + timeout: STORAGE_EXPLORER_CONFIG.timeout, + maxContentLength: Infinity, + maxBodyLength: Infinity, + validateStatus: () => true, + }); + + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + + return response.data; + } catch (error) { + this.logger.error('Storage Explorer API form data proxy error', { + error: error.message, + status: error.response?.status, + path, + storage_url: baseUrl, + method: method.toUpperCase(), + }); + + if (error instanceof HttpException) { + throw error; + } + + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + + throw new HttpException('Internal server error', 500); + } + } +} From 0ce382230009338f72c04e4108004579cf6487c0 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 13 Jan 2026 14:36:43 -0300 Subject: [PATCH 2/6] CHORE: update new endpoint link-table and remove /delete from storage-explorer api --- .../storage-explorer.controller.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts index a18f4a9..34884da 100644 --- a/src/modules/storage-explorer/storage-explorer.controller.ts +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -348,20 +348,19 @@ export class StorageExplorerController { ); } - @ApiOperation({ summary: 'Delete a file from storage' }) - @Delete('storage/delete') + @ApiOperation({ summary: 'Link an existing file in storage to a table' }) + @Post('storage/link-to-table') @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) - async deleteFile( - @Query('file_path') filePath: string, + async linkFileToTable( + @Body() body: any, @User() user: RequestUser, ) { return this.storageExplorerService.proxy( - 'DELETE', - '/storage/delete', + 'POST', + '/storage/link-to-table', user, - undefined, - { file_path: filePath }, + body, ); } From 3b844090038c9d6b2aa6d551cc8a3c4709999e41 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 06:25:58 -0300 Subject: [PATCH 3/6] CHORE: add unique authentication decorator to Storage Explorer controller methods --- .../storage-explorer.controller.ts | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts index 34884da..fd2e4d9 100644 --- a/src/modules/storage-explorer/storage-explorer.controller.ts +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -26,6 +26,7 @@ import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; @ApiTags('Storage Explorer') @Controller('storage-explorer') +@Authenticated() export class StorageExplorerController { private logger: any; @@ -42,7 +43,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' }) @Post('tables/validate-name') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async validateTableName( @Body() body: any, @@ -58,7 +58,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Create a new table' }) @Post('tables') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async createTable( @Body() body: any, @@ -74,7 +73,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'List all tables with pagination' }) @Get('tables') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async listTables( @Query('page') page: number, @@ -91,7 +89,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get table details by ID' }) @Get('tables/:tableId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getTable( @Param('tableId') tableId: string, @@ -106,7 +103,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Link a dataset to a table' }) @Post('tables/:tableId/datasets/:datasetId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async linkDatasetToTable( @Param('tableId') tableId: string, @@ -122,7 +118,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get all datasets linked to a table' }) @Get('tables/:tableId/datasets') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getTableDatasets( @Param('tableId') tableId: string, @@ -137,7 +132,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get table schema' }) @Get('tables/:tableId/schema') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getTableSchema( @Param('tableId') tableId: string, @@ -152,7 +146,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Validate schema compatibility between table and dataset' }) @Post('tables/:tableId/validate-compatibility/:datasetId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async validateSchemaCompatibility( @Param('tableId') tableId: string, @@ -172,7 +165,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get dataset preview data' }) @Get('datasets/:datasetId/preview') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getDatasetPreview( @Param('datasetId') datasetId: string, @@ -190,7 +182,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get dataset schema information' }) @Get('datasets/:datasetId/schema') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getDatasetSchema( @Param('datasetId') datasetId: string, @@ -208,7 +199,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'List all datasets for a specific upload' }) @Get('datasets/upload/:uploadId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async listDatasetsByUpload( @Param('uploadId') uploadId: string, @@ -223,7 +213,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' }) @Put('datasets/:datasetId/refresh-schema') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async refreshDatasetSchema( @Param('datasetId') datasetId: string, @@ -244,7 +233,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'List file explorer uploads with pagination' }) @Get('storage/uploads/history') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async listFileExplorerUploads( @Query('page') page: number, @@ -263,7 +251,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Browse folders and files in storage' }) @Get('storage/browse') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async browseStorage( @Query('path') path: string, @@ -282,7 +269,6 @@ export class StorageExplorerController { @Post('storage/upload/batch') @ApiConsumes('multipart/form-data') @UseInterceptors(FilesInterceptor('files')) - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async batchUpload( @UploadedFiles() files: Array, @@ -317,7 +303,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Create a new folder in storage' }) @Post('storage/folder/create') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async createFolder( @Body() body: any, @@ -333,7 +318,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Download a file from storage' }) @Get('storage/download') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async downloadFile( @Query('file_path') filePath: string, @@ -350,7 +334,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Link an existing file in storage to a table' }) @Post('storage/link-to-table') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async linkFileToTable( @Body() body: any, @@ -366,7 +349,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get detailed file metadata' }) @Get('storage/metadata') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getFileMetadata( @Query('file_path') filePath: string, From 8e63757738524fd85e9303be1cbfc4256f7518f3 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 06:26:21 -0300 Subject: [PATCH 4/6] FEAT: add STORAGE_EXPLORER_API_URL to deployment and values configuration --- deploy/helm-chart/templates/deployment.yaml | 2 ++ deploy/helm-chart/values.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index dc8949d..68b8e1d 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -111,6 +111,8 @@ spec: value: "{{ .Values.maestro.redis_tls }}" - name: PLATFORM_API_URL value: {{ .Values.maestro.platform_api_url }} + - name: STORAGE_EXPLORER_API_URL + value: {{ .Values.maestro.storage_explorer_api_url | quote }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 37cd6be..3c8ebcc 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -47,6 +47,7 @@ maestro: open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com + storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api" dedicated_proxy: "" restricted_ip: "" redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com" From b44d23552c2ee1b4e681f3f4b8f5f6ecb735809c Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 06:40:52 -0300 Subject: [PATCH 5/6] CHORE: remove unecessary header for request --- docsfera.json | 21 ++++++------------- .../storage-explorer.service.ts | 12 ----------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/docsfera.json b/docsfera.json index 92b3f98..ec330d1 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8440,22 +8440,13 @@ ] } }, - "/storage-explorer/storage/delete": { - "delete": { - "operationId": "StorageExplorerController_deleteFile", - "summary": "Delete a file from storage", - "parameters": [ - { - "name": "file_path", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], + "/storage-explorer/storage/link-to-table": { + "post": { + "operationId": "StorageExplorerController_linkFileToTable", + "summary": "Link an existing file in storage to a table", + "parameters": [], "responses": { - "200": { + "201": { "description": "", "content": { "application/json": { diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts index 1430dbb..ba149c0 100644 --- a/src/modules/storage-explorer/storage-explorer.service.ts +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -42,12 +42,6 @@ export class StorageExplorerService { const headers: Record = { 'content-type': 'application/json', - // Forward user context for logging - 'x-customer-id': user.customer_id, - 'x-customer-name': user.customer_name || '', - 'x-user-id': user.user_id || '', - 'x-username': user.username || '', - 'x-customer-tier': user.customer_tier || '', }; this.logger.info('Proxying request to storage-explorer', { @@ -132,12 +126,6 @@ export class StorageExplorerService { } const headers: Record = { - // Forward user context for logging - 'x-customer-id': user.customer_id, - 'x-customer-name': user.customer_name || '', - 'x-user-id': user.user_id || '', - 'x-username': user.username || '', - 'x-customer-tier': user.customer_tier || '', // Let axios set Content-Type for multipart/form-data with boundary ...formData.getHeaders?.(), }; From 3ab2f8f27d0bca1388b910342e4bb5fd0f556148 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 08:51:45 -0300 Subject: [PATCH 6/6] CHORE: remove linkFileToTable endpoint from Storage Explorer controller --- .../storage-explorer.controller.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts index fd2e4d9..ef6a071 100644 --- a/src/modules/storage-explorer/storage-explorer.controller.ts +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -332,21 +332,6 @@ export class StorageExplorerController { ); } - @ApiOperation({ summary: 'Link an existing file in storage to a table' }) - @Post('storage/link-to-table') - @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) - async linkFileToTable( - @Body() body: any, - @User() user: RequestUser, - ) { - return this.storageExplorerService.proxy( - 'POST', - '/storage/link-to-table', - user, - body, - ); - } - @ApiOperation({ summary: 'Get detailed file metadata' }) @Get('storage/metadata') @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)