Merge pull request #443 from dadosfera/feat/qualify

Feat/qualify

deploy/helm-chart/values-stg.yaml

@@ -9,6 +9,7 @@ maestro:
   cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
   redis_database: "1"
   platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
+  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local/api"
 
 hostname: maestro.stg.dadosfera.ai
 

deploy/helm-chart/values.yaml

@@ -47,7 +47,7 @@ maestro:
   open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
   open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
   platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
-  storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api"
+  storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
   dedicated_proxy: ""
   restricted_ip: ""
   redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"

package-lock.json

@@ -17,7 +17,7 @@
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+        "@dadosfera/protospack-v2": "3.38.0-beta.28",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1744,9 +1744,10 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.38.0-beta.26",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz",
-      "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==",
+      "version": "3.38.0-beta.28",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
+      "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -35,7 +35,7 @@
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "^3.38.0-beta.26",
+    "@dadosfera/protospack-v2": "3.38.0-beta.28",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/auth/auth.controller.ts

@@ -55,6 +55,7 @@ import jwt, { JwtPayload } from 'jsonwebtoken';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+import { ApiKeyService } from 'src/modules/api-key/api-key.service';
 
 type CookiesValues = {
   accessToken?: string;
@@ -74,6 +75,7 @@ export class AuthController {
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
     private authClient: AuthClientService,
+    private apiKeyService: ApiKeyService,
   ) {
     this.logger = dadosferaLogger.logger;
 
@@ -477,10 +479,29 @@ export class AuthController {
   async getMe(@Req() req: Request, @Res() res: Response) {
     this.logger.info('GET /auth/me ')
 
+    // Check for API key header first
+    const apiKey = req.get('X-Api-key');
+    if (apiKey) {
+      this.logger.info('Authenticating via X-Api-key header');
+      const { api_key } = await this.apiKeyService.get(apiKey);
+
+      const userDto = {
+        id: api_key.user_id,
+        name: api_key.username,
+        customer: {
+          id: api_key.customer_id,
+          name: api_key.customer_name,
+          tier: api_key.customer_tier,
+        }
+      };
+
+      return res.status(200).json(userDto);
+    }
+
     // Get token and headers
     const accessToken = req.cookies['ddf-auth'];
-    const refreshToken = req.cookies['ddf-refresh-auth']; 
-    const userId = req.cookies['ddf-user-id']; 
+    const refreshToken = req.cookies['ddf-refresh-auth'];
+    const userId = req.cookies['ddf-user-id'];
     const resourceHost = req.headers["host"]
 
     const hasUserSession = Boolean(accessToken) && Boolean(userId);

src/modules/auth/auth.module.ts

@@ -8,10 +8,11 @@ import { AuthClientService } from './auth.service';
 import { DucClient } from '../duc/client.config';
 import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
 import { getOauthSecrets } from 'src/utils/OauthSecrets';
+import { ApiKeyModule } from '../api-key/api-key.module';
 const client = new DucClient();
 
 @Module({
-  imports: [ClientsModule.register([client.providerOptions])],
+  imports: [ClientsModule.register([client.providerOptions]), ApiKeyModule],
   controllers: [AuthController],
   providers: [
     AuthClientService,

src/modules/inputs/dtos/input.model.ts

@@ -14,6 +14,13 @@ export class TableColumns {
   @ApiProperty()
   references: Column[];
   @ApiProperty()
+  identifier_columns: string[];
+  @ApiProperty()
+  destination: Record<'raw' | 'qualify', {
+    table_name: string;
+    table_schema: string;
+  }> | null;
+  @ApiProperty()
   type: string;
 }
 export class AvailableEntity {

src/modules/inputs/inputs.service.ts

@@ -200,7 +200,7 @@ export class InputsService {
   }
 
   async update(id: string, data, info: Info) {
-    this.validateCron({ ...data, info });
+    // this.validateCron({ ...data, info });
     try {
       const updateInputResponse: any = await this.OLD_inputClient.update({
         id,

src/modules/pipelinesV2/interfaces.ts

@@ -1,6 +1,15 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
 import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
+export class PipelineInputsDTO {
+  @ApiProperty()
+  tables: Array<{
+    name: string,
+    type: string,
+    
+  }>
+}
+
 export class IPipelineV2 {
   @ApiProperty()
   id: string;
@@ -123,3 +132,30 @@ export class PipelineFindAllReq {
   @ApiPropertyOptional()
   type?: string | undefined;
 }
+
+export interface UpdateTableDTO {
+  name: string;
+  type: string;
+  columns: string[];
+  destinations: {
+    raw: {
+      table_schema: string;
+      table_name: string;
+    };
+    qualify: {
+      table_schema: string;
+      table_name: string;
+    };
+  };
+  identifier_columns: string[];
+  reference_column: {
+    name: string;
+    type: string;
+  };
+  memory: number;
+}
+
+export interface UpdatePlatformInputRequest {
+  cron: string;
+  tables: Array<UpdateTableDTO>;
+}

src/modules/pipelinesV2/pipelines.controller.ts

@@ -43,11 +43,15 @@ import {
   IPipelineV2,
   IInitUploadCSVFile,
   PipelineFindAllReq,
+  UpdatePlatformInputRequest,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+import { TableColumns } from '../inputs/dtos/input.model';
+import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 
 @ApiTags('PipelinesV2')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -223,6 +227,7 @@ export class PipelinesController {
       .then((res) => {
         //{pipeline:{tables: {tables: [], input_id: ''}}}
         let tables = JSON.parse(res.pipeline.config.tables);
+        const input_id = tables?.input_id;
         if (tables?.tables) tables = tables.tables;
         Object.assign(res.pipeline, {
           transformations: res.pipeline.transformations
@@ -231,6 +236,7 @@ export class PipelinesController {
           config: {
             cron: res.pipeline.config.cron,
             tables,
+            input_id
           },
           properties: res.pipeline.properties
             ? JSON.parse(res.pipeline.properties)
@@ -277,6 +283,45 @@ export class PipelinesController {
     return response;
   }
 
+  @Patch('/:pipelineId/inputs/:id')
+  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  async updatePipelineInput(
+    @Language() language: LanguageEnum,
+    @Body() pipelineInputDTO: UpdatePlatformInputRequest,
+    @Param('id') inputId: string,
+    @Param('pipelineId') pipelineId: string,
+    @User() user: RequestUser,
+  ) {
+    this.logger.info('PipelinesController - update', { user });
+
+
+    const { customer_id, customer_name, user_id, username } = user;
+    const info: Info = {
+      user_id: user.user_id,
+      customer: user.customer_name,
+      customer_id: user.customer_id,
+    };
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    const response = await this.pipelinesClientService.updatePipelineInput(
+      pipelineId,
+      inputId,
+      pipelineInputDTO,
+      info,
+      user,
+      metadata,
+    );
+
+    this.logger.info('PipelinesController - update: OK', { user });
+    return response;
+  }
+
   @ApiInternalOnlyEndpoint()
   @Put('/:id')
   @ApiOperation({

src/modules/pipelinesV2/pipelines.module.ts

@@ -11,6 +11,7 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe
 import { ConnectorModule } from '../connector/connector.module';
 import { InputsModule } from '../inputs/inputs.module';
 import { TransformationsModule } from '../transformations/transformations.module';
+import { PlatformApiModule } from '../platform-api/platform-api.module';
 
 const client = new PipelinesClientConfiguration();
 
@@ -21,6 +22,7 @@ const client = new PipelinesClientConfiguration();
     ConnectorModule,
     InputsModule,
     TransformationsModule,
+    PlatformApiModule
   ],
   controllers: [PipelinesController],
   providers: [PipelinesService, DadosferaLogger],

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-async-promise-executor */
 import {
   BadRequestException,
   HttpException,
@@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
 
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PipelinesClientConfiguration } from './pipelines-client';
-import { ICreatePipelineV2Req } from './interfaces';
+import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
 import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
@@ -26,6 +27,8 @@ import { TransformationsService } from '../transformations/transformations.servi
 import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
 import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
+import { PlatformApiService } from '../platform-api/platform-api.service';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -39,6 +42,7 @@ export class PipelinesService implements OnModuleInit {
     private readonly connectorService: ConnectorClientService,
     private readonly inputsService: InputsService,
     private readonly transformationsService: TransformationsService,
+    private readonly platformAPI: PlatformApiService
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -138,6 +142,7 @@ export class PipelinesService implements OnModuleInit {
     const findOnePipelineResponse = await lastValueFrom(
       this.pipelineReadService.PipelineV2FindOne(data, metadata),
     );
+    console.log('pipeline find one response', findOnePipelineResponse);
     this.logger.info('Done');
 
     return findOnePipelineResponse;
@@ -339,4 +344,137 @@ export class PipelinesService implements OnModuleInit {
 
     return res;
   }
+
+  async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
+    this.logger.info('InputClientService - Update');
+
+    this.logger.info('Update Dynamo Reference');
+    const pipelineIdFormat = pipelineId.split('-').join('_');
+    const updateInputResponse = await this.inputsService.update(
+      inputId,
+      updateInputDTO,
+      info
+    )
+
+    const requests = [];
+
+    this.logger.info('Dynamo Response', updateInputResponse);
+
+    for (const [index, table] of updateInputDTO.tables.entries()) {
+      const id = `${pipelineIdFormat}_${index}`;
+      this.logger.info('Updating input reference for table', table.name);
+      const body = {}
+      if (table.columns) {
+        body['column_include_list'] = table.columns;
+      }
+
+      if (table.reference_column) {
+        body['incremental_column_name'] = table.reference_column.name;
+        body['incremental_column_type'] = table.reference_column.type;
+      }
+
+      if (table.identifier_columns) {
+        body['primary_keys'] = table.identifier_columns;
+      }
+
+      this.logger.info('Request body', body);
+      const updateCollumns = this.platformAPI.proxy(
+        'PATCH',
+        `/jobs/${id}/input`,
+        user,
+        body
+      )
+      requests.push(updateCollumns);
+
+      if (table.memory) {
+        this.logger.info('Updating memory allocation for table', table.name);
+        const updateMemory = this.platformAPI.proxy(
+          'PUT',
+          `/jobs/${id}/memory`,
+          user,
+          {
+            amount: table.memory
+          }
+        )
+        requests.push(updateMemory);
+      }
+
+      if (table.type) {
+          const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
+          requests.push(updateSyncMode);
+      }
+    }
+
+    this.logger.info('Create Platform Request for each JOB');
+
+    if (updateInputDTO.cron) {
+      const crnUpdatedRequest = new Promise(async (resolve, reject) => {
+        const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
+
+        if (response.error) {
+          this.logger.error('Error updating pipeline cron', response.error);
+          return reject(new ErrorBuilder(response.error));
+        }
+        this.logger.error('Pipeline cron updated successfully', response);
+        return resolve(response);
+      });
+      requests.push(crnUpdatedRequest);
+    }
+
+    this.logger.info('Executing all request for the platform api');
+
+    const results = await Promise.allSettled(requests);
+    this.logger.info('Platform api response', results);
+
+    return updateInputResponse;
+    
+  }
+
+  private async updatePipelineSyncMode(table: UpdateTableDTO,  pipelineId: string, user: RequestUser) {
+    const body = {
+      target_load_type: table.type
+    }
+
+    if (table.type === 'incremental_with_qualify') {
+      body['incremental_column_name'] = table.reference_column.name;
+      body['incremental_column_type'] = table.reference_column.type;
+      body['primary_keys'] = table.identifier_columns;
+    }
+
+    if (table.type === 'incremental') {
+      body['incremental_column_name'] = table.reference_column.name;
+      body['incremental_column_type'] = table.reference_column.type;
+    }
+
+    this.logger.info('Updating pipeline sync mode', {
+      pipelineId,
+      body
+    });
+
+    return this.platformAPI.proxy(
+      "POST",
+      `/jobs/jdbc/${pipelineId}/sync-mode`,
+      user,
+      body
+    )
+  }
+
+  private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
+    try {
+      const response = await this.platformAPI.proxy(
+        'PATCH',
+        `/pipeline/${pipelineId}`,
+        user,
+        {
+          cron
+        }
+      );
+
+      return response
+    } catch (error) {
+        return {
+          error: error.message
+        }
+    }
+  }
 }

src/modules/platform-api/platform-api.controller.ts

@@ -25,6 +25,15 @@ import { ElasticsearchService } from '../../services/elasticsearch';
 import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
 import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
+import { ValidationTableDTO } from './platform-api.dto';
+
+
+type ValidateTablesDTO = {
+  tables: Array<{
+    table_schema: string,
+    table_name: string
+  }>
+}
 
 @ApiTags('Platform API')
 @Controller('platform')
@@ -720,6 +729,41 @@ export class PlatformApiController {
     );
   }
 
+  // ==================== PIPELINE VALIDATION ====================
+
+  @Get('pipelines/catalog/schemas')
+  @ApiOperation({ summary: 'Get available schemas' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getAvailableSchemas(
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'GET',
+      `/catalog/schemas`,
+      user,
+      undefined,
+      query,
+    );
+  }
+
+  @Post('pipelines/catalog/tables/validate')
+  @ApiOperation({ summary: 'Validate Table and Schema' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async validateTableAndSchema(
+    @Body() payload: ValidationTableDTO,
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'POST',
+      `/catalog/tables/validate`,
+      user,
+      payload,
+      query,
+    );
+  }
+
   // ==================== PIPELINE RUN ROUTES ====================
 
   @Get('pipeline/:pipelineId/pipeline_run')

src/modules/platform-api/platform-api.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ValidationTableDTO {
+    @ApiProperty()
+    tables: Array<{
+        table_name: string;
+        table_schema: string;
+    }>
+}

src/modules/storage-explorer/storage-explorer.config.ts

@@ -1,13 +1,13 @@
 export const STORAGE_EXPLORER_CONFIG = {
-  getUrl: (customerId: string): string => {
+  getUrl: (customerName: string): string => {
     const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
     if (!urlTemplate) {
       throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
     }
     // Replace {customer_id} placeholder with actual customer ID
     // For local: http://172.17.0.1:8000/api (no placeholder)
-    // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api
-    return urlTemplate.replace('{customer_id}', customerId);
+    // For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api
+    return urlTemplate.replace('{customer}', customerName);
   },
   timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
 };

src/modules/storage-explorer/storage-explorer.controller.ts

@@ -3,13 +3,13 @@ import {
   Get,
   Post,
   Put,
-  Delete,
   Param,
   Body,
   Query,
   Inject,
   UseInterceptors,
   UploadedFiles,
+  Headers,
 } from '@nestjs/common';
 import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
 import { FilesInterceptor } from '@nestjs/platform-express';
@@ -47,11 +47,13 @@ export class StorageExplorerController {
   async validateTableName(
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/tables/validate-name',
       user,
+      
       body,
     );
   }
@@ -62,11 +64,13 @@ export class StorageExplorerController {
   async createTable(
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/tables/',
       user,
+      
       body,
     );
   }
@@ -77,11 +81,13 @@ export class StorageExplorerController {
   async listTables(
     @Query('page') page: number,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/tables/',
       user,
+      
       undefined,
       { page },
     );
@@ -93,6 +99,7 @@ export class StorageExplorerController {
   async getTable(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
@@ -108,11 +115,13 @@ export class StorageExplorerController {
     @Param('tableId') tableId: string,
     @Param('datasetId') datasetId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       `/tables/${tableId}/datasets/${datasetId}`,
       user,
+      
     );
   }
 
@@ -122,11 +131,13 @@ export class StorageExplorerController {
   async getTableDatasets(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/tables/${tableId}/datasets`,
       user,
+      
     );
   }
 
@@ -136,11 +147,13 @@ export class StorageExplorerController {
   async getTableSchema(
     @Param('tableId') tableId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/tables/${tableId}/schema`,
       user,
+      
     );
   }
 
@@ -156,6 +169,7 @@ export class StorageExplorerController {
       'POST',
       `/tables/${tableId}/validate-compatibility/${datasetId}`,
       user,
+      
     );
   }
 
@@ -170,11 +184,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Query('limit') limit: number,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/${datasetId}/preview`,
       user,
+      
       undefined,
       { limit },
     );
@@ -187,11 +203,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Query('force_refresh') forceRefresh: boolean,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/${datasetId}/schema`,
       user,
+      
       undefined,
       { force_refresh: forceRefresh },
     );
@@ -203,11 +221,13 @@ export class StorageExplorerController {
   async listDatasetsByUpload(
     @Param('uploadId') uploadId: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       `/datasets/upload/${uploadId}`,
       user,
+      
     );
   }
 
@@ -218,11 +238,13 @@ export class StorageExplorerController {
     @Param('datasetId') datasetId: string,
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'PUT',
       `/datasets/${datasetId}/refresh-schema`,
       user,
+      
       body,
     );
   }
@@ -239,11 +261,13 @@ export class StorageExplorerController {
     @Query('limit') limit: number,
     @Query('folder_path') folderPath: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/uploads/history',
       user,
+      
       undefined,
       { page, limit, folder_path: folderPath },
     );
@@ -255,11 +279,13 @@ export class StorageExplorerController {
   async browseStorage(
     @Query('path') path: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/browse',
       user,
+      
       undefined,
       { path },
     );
@@ -274,6 +300,7 @@ export class StorageExplorerController {
     @UploadedFiles() files: Array<Express.Multer.File>,
     @Body('folder_path') folderPath: string,
     @User() user: RequestUser,
+    
   ) {
     // Create FormData to forward files to storage-explorer API
     const formData = new FormData();
@@ -297,6 +324,7 @@ export class StorageExplorerController {
       'POST',
       '/storage/upload/batch',
       user,
+      
       formData,
     );
   }
@@ -307,11 +335,13 @@ export class StorageExplorerController {
   async createFolder(
     @Body() body: any,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'POST',
       '/storage/folder/create',
       user,
+      
       body,
     );
   }
@@ -322,11 +352,13 @@ export class StorageExplorerController {
   async downloadFile(
     @Query('file_path') filePath: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',
       '/storage/download',
       user,
+      
       undefined,
       { file_path: filePath },
     );
@@ -338,6 +370,7 @@ export class StorageExplorerController {
   async getFileMetadata(
     @Query('file_path') filePath: string,
     @User() user: RequestUser,
+    
   ) {
     return this.storageExplorerService.proxy(
       'GET',

src/modules/storage-explorer/storage-explorer.service.ts

@@ -20,7 +20,7 @@ export class StorageExplorerService {
     path: string,
     user: RequestUser,
     body?: any,
-    query?: Record<string, any>,
+    query?: Record<string, any>
   ): Promise<any> {
     // Validate customer_id is present for multi-tenant isolation
     if (!user.customer_id) {
@@ -28,7 +28,7 @@ export class StorageExplorerService {
     }
 
     // Get customer-specific storage-explorer URL
-    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
+    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
     const url = new URL(`${baseUrl}${path}`);
 
     // Add query params

src/pipes/object-validation.pipe.ts

@@ -14,7 +14,10 @@ export class ValidationPipe implements PipeTransform<any> {
       return value;
     }
     const object = plainToInstance(metatype, value);
-    const errors = await validate(object);
+    const errors = await validate(object, {
+      forbidUnknownValues: false,
+      whitelist: true,
+    });
     if (errors.length > 0) {
       const errorMessages = errors.map((err) => err.constraints);
       throw new BadRequestException(errorMessages);