Compare commits

..
Author SHA1 Message Date
RafaelandClaude Opus 4.6 051fb6e4dd FIX: uppercase table_name and table_schema in Nimbus rename calls
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-16 10:39:55 -03:00
RafaelandClaude Opus 4.6 e3099aa2b2 FIX: uppercase table_name and table_schema in ES lookup
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-16 10:17:56 -03:00
RafaelandClaude Opus 4.6 12c61d9b5d FIX: rename-tables proxy path and ES lookup
- Fix proxy path: /jobs/jdbc/:jobId/rename-tables → /jobs/:jobId/rename-tables
- Remove pipeline_id from ES data asset lookup, search by table_name + table_schema only

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-13 18:08:39 -03:00
Rafael 99c2a9ecf5 UPDATE: force deployment of maestro 2026-03-12 18:02:49 -03:00
Rafael Santana c0f75d241f Merge pull request #449 from dadosfera/feat/rename-tables-catalog-sync
Feat/rename tables catalog sync
2026-03-12 17:57:44 -03:00
Rafael Santana 1e0fb78dff Merge branch 'beta' into feat/rename-tables-catalog-sync 2026-03-12 17:57:37 -03:00
Marcos Rodrigues Silva b70d37423d Merge pull request #454 from dadosfera/fix/header-validation
FIX: types
2026-03-11 10:27:31 -03:00
Marcos Rodrigues Silva 3bcbba9581 Merge pull request #453 from dadosfera/fix/header-validation
Fix/header validation
2026-03-10 17:35:31 -03:00
Marcos Rodrigues Silva 6f9c967c96 Merge pull request #452 from dadosfera/feat/qualify
FIX: stringify headers
2026-03-10 17:17:48 -03:00
marcos-silva-rodrigues b8bdc5beea FIX: stringify headers 2026-03-10 16:00:39 -03:00
Marcos Rodrigues Silva 269f70b309 Merge pull request #451 from dadosfera/feat/qualify
FIX: ghost commit
2026-03-10 15:22:36 -03:00
marcos-silva-rodrigues 93f452ae05 FIX: ghost commit 2026-03-10 15:19:18 -03:00
Marcos Rodrigues Silva e39378229f Merge pull request #450 from dadosfera/feat/qualify
feat: list headers
2026-03-10 14:43:43 -03:00
marcos-silva-rodrigues 61a4f724ef feat: list headers 2026-03-10 14:43:11 -03:00
RafaelandClaude Opus 4.6 38a9e21f5f feat: add rename-tables endpoint with catalog sync and rollback
Add POST /platform/jobs/:jobId/rename-tables that renames Snowflake
tables via platform-api and syncs the rename to Elasticsearch and
Nimbus (table-metadata, column-metadata, data-preview). If catalog
sync fails, all completed catalog steps are rolled back in reverse
order and the Snowflake rename is reverted.

- Support any connector type (jdbc, singer, s3) via getJobByAnyConnectorType
- Resolve old table names from output_config (raw/qualify)
- Skip qualify sync when output_config.qualify has no table_name
- Add findDataAssetByPipelineAndTable and updateDataAsset to ElasticsearchService
- Add renameTableOnNimbus, renameColumnMetadataOnNimbus, renameDataPreviewOnNimbus to CatalogService
- Add upstream error logging to PlatformApiService

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 17:29:12 -03:00
Marcos Rodrigues Silva d25bfd147c Merge pull request #447 from dadosfera/fix/proxy-urls
Fix/proxy urls
2026-02-20 14:32:50 -03:00
Marcos Rodrigues Silva 6c57bac235 Merge pull request #445 from dadosfera/feat/qualify
FIX: storage url
2026-02-18 15:43:13 -03:00
marcos-silva-rodrigues cf8eed35a3 FIX: storage url 2026-02-18 15:32:05 -03:00
Marcos Rodrigues Silva 55fc85c544 Merge pull request #444 from dadosfera/feat/qualify
FIX: update storage port
2026-02-16 18:12:27 -03:00
marcos-silva-rodrigues 72ed637640 FIX: update storage port 2026-02-16 18:11:49 -03:00
Marcos Rodrigues Silva 890364f597 Merge pull request #443 from dadosfera/feat/qualify
Feat/qualify
2026-02-13 11:27:41 -03:00
marcos-silva-rodrigues 30eec733b2 FEAT: update proto 2026-02-13 10:01:46 -03:00
marcos-silva-rodrigues 30a41ba144 FEAT: qualify route update 2026-02-12 17:48:05 -03:00
Marcos Rodrigues Silva 2dc032e7e7 Merge pull request #442 from dadosfera/feat/qualify
FIX: platform routes
2026-02-05 10:33:39 -03:00
marcos-silva-rodrigues 7f5981731f merge 2026-02-05 10:32:07 -03:00
marcos-silva-rodrigues 66309c7bbe FIX: platform routes 2026-02-05 10:29:26 -03:00
Marcos Rodrigues Silva 15048eaf8a Merge pull request #441 from dadosfera/feat/qualify
FEAT: qualify contract
2026-02-04 17:09:19 -03:00
marcos-silva-rodrigues 0e169a3cbc FIX: protospack version 2026-02-04 16:50:20 -03:00
marcos-silva-rodrigues b5d933eaf3 FEAT: qualify contract 2026-02-04 16:43:49 -03:00
Marcos Rodrigues Silva 6fa9bf861a Merge pull request #440 from dadosfera/fix/proxy-urls
FIX: storage service dns
2026-02-02 16:38:00 -03:00
Marcos Rodrigues Silva e08734c97f Merge pull request #439 from dadosfera/fix/proxy-urls
FIX: send token by storage
2026-02-02 15:39:00 -03:00
Marcos Rodrigues Silva 54b75ce11b Merge pull request #438 from dadosfera/fix/proxy-urls
FIX: storage api
2026-02-02 14:43:00 -03:00
Marcos Rodrigues Silva 85234fe0dd Merge pull request #437 from dadosfera/fix/proxy-urls
Fix/proxy urls
2026-02-02 14:28:04 -03:00
18 changed files with 1940 additions and 1312 deletions
+1
View File
@@ -4,6 +4,7 @@
# Maestro
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
```mermaid
+1330 -1280
View File
File diff suppressed because it is too large Load Diff
+5 -4
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
"@dadosfera/protospack-v2": "3.38.0-beta.28",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1744,9 +1744,10 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.26",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz",
"integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==",
"version": "3.38.0-beta.28",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
"integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
+1 -1
View File
@@ -35,7 +35,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
"@dadosfera/protospack-v2": "3.38.0-beta.28",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
+1
View File
@@ -111,3 +111,4 @@ function configureSwagger(app: INestApplication) {
);
}
bootstrap();
+1
View File
@@ -478,6 +478,7 @@ export class AuthController {
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
this.logger.info(JSON.stringify(req.headers));
// Check for API key header first
const apiKey = req.get('X-Api-key');
+58
View File
@@ -734,6 +734,64 @@ class CatalogService implements OnModuleInit {
}
}
async renameTableOnNimbus(
nimbusUrl: string,
nimbusId: number,
changes: { table_name?: string; table_schema?: string; display_name?: string },
): Promise<void> {
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
await axios.patch(endpoint, changes);
}
async renameColumnMetadataOnNimbus(
nimbusUrl: string,
databaseName: string,
oldTableName: string,
oldTableSchema: string,
newTableName: string,
newTableSchema: string,
): Promise<void> {
const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
const { data: columns } = await axios.get(listEndpoint);
const filtered = Array.isArray(columns) ? columns : [];
for (const column of filtered) {
const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
await axios.patch(patchEndpoint, {
table_name: newTableName,
table_schema: newTableSchema,
});
}
this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
}
async renameDataPreviewOnNimbus(
nimbusUrl: string,
databaseName: string,
oldTableName: string,
oldTableSchema: string,
newTableName: string,
newTableSchema: string,
): Promise<void> {
const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
const { data: previews } = await axios.get(listEndpoint);
const filtered = Array.isArray(previews) ? previews : [];
for (const preview of filtered) {
const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
await axios.patch(patchEndpoint, {
table_name: newTableName,
table_schema: newTableSchema,
});
}
this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
+7
View File
@@ -14,6 +14,13 @@ export class TableColumns {
@ApiProperty()
references: Column[];
@ApiProperty()
identifier_columns: string[];
@ApiProperty()
destination: Record<'raw' | 'qualify', {
table_name: string;
table_schema: string;
}> | null;
@ApiProperty()
type: string;
}
export class AvailableEntity {
+1 -1
View File
@@ -200,7 +200,7 @@ export class InputsService {
}
async update(id: string, data, info: Info) {
this.validateCron({ ...data, info });
// this.validateCron({ ...data, info });
try {
const updateInputResponse: any = await this.OLD_inputClient.update({
id,
+36
View File
@@ -1,6 +1,15 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export class PipelineInputsDTO {
@ApiProperty()
tables: Array<{
name: string,
type: string,
}>
}
export class IPipelineV2 {
@ApiProperty()
id: string;
@@ -123,3 +132,30 @@ export class PipelineFindAllReq {
@ApiPropertyOptional()
type?: string | undefined;
}
export interface UpdateTableDTO {
name: string;
type: string;
columns: string[];
destinations: {
raw: {
table_schema: string;
table_name: string;
};
qualify: {
table_schema: string;
table_name: string;
};
};
identifier_columns: string[];
reference_column: {
name: string;
type: string;
};
memory: number;
}
export interface UpdatePlatformInputRequest {
cron: string;
tables: Array<UpdateTableDTO>;
}
@@ -43,11 +43,15 @@ import {
IPipelineV2,
IInitUploadCSVFile,
PipelineFindAllReq,
UpdatePlatformInputRequest,
} from './interfaces';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { TableColumns } from '../inputs/dtos/input.model';
import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@ApiTags('PipelinesV2')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -223,6 +227,7 @@ export class PipelinesController {
.then((res) => {
//{pipeline:{tables: {tables: [], input_id: ''}}}
let tables = JSON.parse(res.pipeline.config.tables);
const input_id = tables?.input_id;
if (tables?.tables) tables = tables.tables;
Object.assign(res.pipeline, {
transformations: res.pipeline.transformations
@@ -231,6 +236,7 @@ export class PipelinesController {
config: {
cron: res.pipeline.config.cron,
tables,
input_id
},
properties: res.pipeline.properties
? JSON.parse(res.pipeline.properties)
@@ -277,6 +283,45 @@ export class PipelinesController {
return response;
}
@Patch('/:pipelineId/inputs/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async updatePipelineInput(
@Language() language: LanguageEnum,
@Body() pipelineInputDTO: UpdatePlatformInputRequest,
@Param('id') inputId: string,
@Param('pipelineId') pipelineId: string,
@User() user: RequestUser,
) {
this.logger.info('PipelinesController - update', { user });
const { customer_id, customer_name, user_id, username } = user;
const info: Info = {
user_id: user.user_id,
customer: user.customer_name,
customer_id: user.customer_id,
};
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
language,
});
const response = await this.pipelinesClientService.updatePipelineInput(
pipelineId,
inputId,
pipelineInputDTO,
info,
user,
metadata,
);
this.logger.info('PipelinesController - update: OK', { user });
return response;
}
@ApiInternalOnlyEndpoint()
@Put('/:id')
@ApiOperation({
@@ -11,6 +11,7 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe
import { ConnectorModule } from '../connector/connector.module';
import { InputsModule } from '../inputs/inputs.module';
import { TransformationsModule } from '../transformations/transformations.module';
import { PlatformApiModule } from '../platform-api/platform-api.module';
const client = new PipelinesClientConfiguration();
@@ -21,6 +22,7 @@ const client = new PipelinesClientConfiguration();
ConnectorModule,
InputsModule,
TransformationsModule,
PlatformApiModule
],
controllers: [PipelinesController],
providers: [PipelinesService, DadosferaLogger],
+139 -1
View File
@@ -1,3 +1,4 @@
/* eslint-disable no-async-promise-executor */
import {
BadRequestException,
HttpException,
@@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PipelinesClientConfiguration } from './pipelines-client';
import { ICreatePipelineV2Req } from './interfaces';
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service';
@@ -26,6 +27,8 @@ import { TransformationsService } from '../transformations/transformations.servi
import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import { PlatformApiService } from '../platform-api/platform-api.service';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
export class PipelinesService implements OnModuleInit {
logger: DadosferaLogger;
@@ -39,6 +42,7 @@ export class PipelinesService implements OnModuleInit {
private readonly connectorService: ConnectorClientService,
private readonly inputsService: InputsService,
private readonly transformationsService: TransformationsService,
private readonly platformAPI: PlatformApiService
) {
this.logger = dadosferaLogger.logger;
}
@@ -138,6 +142,7 @@ export class PipelinesService implements OnModuleInit {
const findOnePipelineResponse = await lastValueFrom(
this.pipelineReadService.PipelineV2FindOne(data, metadata),
);
console.log('pipeline find one response', findOnePipelineResponse);
this.logger.info('Done');
return findOnePipelineResponse;
@@ -339,4 +344,137 @@ export class PipelinesService implements OnModuleInit {
return res;
}
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
this.logger.info('InputClientService - Update');
this.logger.info('Update Dynamo Reference');
const pipelineIdFormat = pipelineId.split('-').join('_');
const updateInputResponse = await this.inputsService.update(
inputId,
updateInputDTO,
info
)
const requests = [];
this.logger.info('Dynamo Response', updateInputResponse);
for (const [index, table] of updateInputDTO.tables.entries()) {
const id = `${pipelineIdFormat}_${index}`;
this.logger.info('Updating input reference for table', table.name);
const body = {}
if (table.columns) {
body['column_include_list'] = table.columns;
}
if (table.reference_column) {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
}
if (table.identifier_columns) {
body['primary_keys'] = table.identifier_columns;
}
this.logger.info('Request body', body);
const updateCollumns = this.platformAPI.proxy(
'PATCH',
`/jobs/${id}/input`,
user,
body
)
requests.push(updateCollumns);
if (table.memory) {
this.logger.info('Updating memory allocation for table', table.name);
const updateMemory = this.platformAPI.proxy(
'PUT',
`/jobs/${id}/memory`,
user,
{
amount: table.memory
}
)
requests.push(updateMemory);
}
if (table.type) {
const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
requests.push(updateSyncMode);
}
}
this.logger.info('Create Platform Request for each JOB');
if (updateInputDTO.cron) {
const crnUpdatedRequest = new Promise(async (resolve, reject) => {
const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
if (response.error) {
this.logger.error('Error updating pipeline cron', response.error);
return reject(new ErrorBuilder(response.error));
}
this.logger.error('Pipeline cron updated successfully', response);
return resolve(response);
});
requests.push(crnUpdatedRequest);
}
this.logger.info('Executing all request for the platform api');
const results = await Promise.allSettled(requests);
this.logger.info('Platform api response', results);
return updateInputResponse;
}
private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) {
const body = {
target_load_type: table.type
}
if (table.type === 'incremental_with_qualify') {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
body['primary_keys'] = table.identifier_columns;
}
if (table.type === 'incremental') {
body['incremental_column_name'] = table.reference_column.name;
body['incremental_column_type'] = table.reference_column.type;
}
this.logger.info('Updating pipeline sync mode', {
pipelineId,
body
});
return this.platformAPI.proxy(
"POST",
`/jobs/jdbc/${pipelineId}/sync-mode`,
user,
body
)
}
private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
try {
const response = await this.platformAPI.proxy(
'PATCH',
`/pipeline/${pipelineId}`,
user,
{
cron
}
);
return response
} catch (error) {
return {
error: error.message
}
}
}
}
@@ -10,6 +10,7 @@ import {
Query,
Inject,
BadRequestException,
HttpException,
} from '@nestjs/common';
import { ApiTags, ApiOperation } from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -25,6 +26,9 @@ import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
import { CatalogService } from '../catalog/catalog.service';
import { PackTheMetadata } from '../../utils/PackTheMetadata';
import { ValidationTableDTO } from './platform-api.dto';
type ValidateTablesDTO = {
@@ -34,6 +38,11 @@ type ValidateTablesDTO = {
}>
}
type RenameTablesBody = {
raw?: { table_name: string; table_schema: string };
qualify?: { table_name: string; table_schema: string };
}
@ApiTags('Platform API')
@Controller('platform')
export class PlatformApiController {
@@ -44,6 +53,7 @@ export class PlatformApiController {
private readonly elasticsearchService: ElasticsearchService,
private readonly dynamoDBService: DynamoDBService,
private readonly customersService: CustomersService,
private readonly catalogService: CatalogService,
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
@@ -75,6 +85,23 @@ export class PlatformApiController {
return jobId?.replace(/-/g, '_') || '';
}
private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise<any> {
const connectorTypes = ['jdbc', 'singer', 's3'];
for (const type of connectorTypes) {
try {
const job = await this.platformApiService.proxy(
'GET',
`/jobs/${type}/${normalizedJobId}`,
user,
);
return job;
} catch (error) {
// Continue to next connector type
}
}
throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404);
}
/**
* Extract the pipeline ID (base UUID) from a job ID.
* Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1").
@@ -728,26 +755,10 @@ export class PlatformApiController {
);
}
// ==================== Catalog ROUTES ====================
@Get('pipelines/catalog/tables')
@ApiOperation({ summary: 'Get all tables available' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getAvailableTables(
@User() user: RequestUser,
@Query() query: Record<string, string>,
) {
return this.platformApiService.proxy(
'GET',
'/catalog/tables',
user,
undefined,
query,
);
}
// ==================== PIPELINE VALIDATION ====================
@Get('pipelines/catalog/schemas')
@ApiOperation({ summary: 'Get all schemas available' })
@ApiOperation({ summary: 'Get available schemas' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getAvailableSchemas(
@User() user: RequestUser,
@@ -755,7 +766,7 @@ export class PlatformApiController {
) {
return this.platformApiService.proxy(
'GET',
'/catalog/schemas',
`/catalog/schemas`,
user,
undefined,
query,
@@ -763,18 +774,18 @@ export class PlatformApiController {
}
@Post('pipelines/catalog/tables/validate')
@ApiOperation({ summary: 'Validate tables and schemas' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
@ApiOperation({ summary: 'Validate Table and Schema' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async validateTableAndSchema(
@Body() payload: ValidationTableDTO,
@User() user: RequestUser,
@Query() query: Record<string, string>,
@Body() validateTablesDto: ValidateTablesDTO[]
) {
return this.platformApiService.proxy(
'POST',
'/catalog/tables/validate',
`/catalog/tables/validate`,
user,
validateTablesDto,
payload,
query,
);
}
@@ -968,6 +979,192 @@ export class PlatformApiController {
return result;
}
@Post('jobs/:jobId/rename-tables')
@ApiOperation({ summary: 'Rename job output tables and sync to catalog' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async renameJobTables(
@Param('jobId') jobId: string,
@Body() body: RenameTablesBody,
@User() user: RequestUser,
) {
const normalizedJobId = this.normalizeJobId(jobId);
const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user);
const result = await this.platformApiService.proxy(
'POST',
`/jobs/${normalizedJobId}/rename-tables`,
user,
body,
);
try {
await this.syncTableRenameToCatalog(jobId, body, currentJob, user);
} catch (error) {
this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message });
const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {});
if (reverseBody) {
try {
await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody);
this.logger.info('Snowflake rename rolled back', { jobId });
} catch (rollbackError) {
this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message });
}
}
throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500);
}
return result;
}
private buildSnowflakeRollbackBody(
body: RenameTablesBody,
outputConfig: any,
): RenameTablesBody | null {
const reverse: RenameTablesBody = {};
if (body.raw) {
const nested = outputConfig.raw;
const oldTableName = nested?.table_name || outputConfig.table_name;
const oldTableSchema = nested?.table_schema || 'PUBLIC';
if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema };
}
if (body.qualify) {
const nested = outputConfig.qualify;
if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' };
}
return Object.keys(reverse).length > 0 ? reverse : null;
}
/**
* Sync table rename to Elasticsearch and Nimbus.
*
* For each target (raw, qualify):
* 1. Resolve old table name from output_config
* 2. Find the ES data asset by pipeline + table + schema
* 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview
* 4. If any step fails, rollback all completed steps for that target
*/
private async syncTableRenameToCatalog(
jobId: string,
body: RenameTablesBody,
currentJob: any,
user: RequestUser,
): Promise<void> {
const pipelineId = this.extractPipelineIdFromJobId(jobId);
const outputConfig = currentJob.output_config || {};
const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } });
const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`;
const targets = this.buildRenameTargets(body, outputConfig);
for (const { key, oldTableName, oldTableSchema, newValues } of targets) {
const rollbackSteps: Array<() => Promise<void>> = [];
try {
const dataAsset = await this.elasticsearchService.findDataAssetByTable(
user.customer_name, oldTableName, oldTableSchema,
);
if (!dataAsset) {
this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema });
continue;
}
const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset;
const oldValues = { table_name: oldTableName, table_schema: oldTableSchema };
// ES update
const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name };
await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields);
rollbackSteps.push(() => this.elasticsearchService.updateDataAsset(
user.customer_name, esAssetId,
{ name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName },
));
const newTableNameUpper = newValues.table_name.toUpperCase();
const newTableSchemaUpper = newValues.table_schema.toUpperCase();
const oldTableNameUpper = oldTableName.toUpperCase();
const oldTableSchemaUpper = oldTableSchema.toUpperCase();
// Nimbus table-metadata
if (nimbusId) {
await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper });
rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper }));
}
// Nimbus column-metadata
await this.catalogService.renameColumnMetadataOnNimbus(
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
);
rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus(
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
));
// Nimbus data-preview
await this.catalogService.renameDataPreviewOnNimbus(
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
);
rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus(
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
));
this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name });
} catch (error) {
this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message });
await this.executeRollback(rollbackSteps, key, jobId);
throw error;
}
}
}
private buildRenameTargets(
body: RenameTablesBody,
outputConfig: any,
): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> {
const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' };
const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = [];
for (const key of ['raw', 'qualify'] as const) {
if (!body[key]) continue;
const nested = outputConfig[key];
// qualify: only sync if output_config.qualify already exists
if (key === 'qualify' && !nested?.table_name) continue;
const oldTableName = nested?.table_name || outputConfig.table_name;
if (!oldTableName) continue;
targets.push({
key,
oldTableName,
oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key],
newValues: body[key],
});
}
return targets;
}
private async executeRollback(
steps: Array<() => Promise<void>>,
targetKey: string,
jobId: string,
): Promise<void> {
for (const rollback of steps.reverse()) {
try {
await rollback();
} catch (error) {
this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message });
}
}
}
@Get('jobs/jdbc/configs/allowed_datatypes')
@ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
@@ -0,0 +1,9 @@
import { ApiProperty } from "@nestjs/swagger";
export class ValidationTableDTO {
@ApiProperty()
tables: Array<{
table_name: string;
table_schema: string;
}>
}
@@ -7,9 +7,10 @@ import { PlatformApiService } from './platform-api.service';
import { ElasticsearchModule } from '../../services/elasticsearch';
import { DynamoDBModule } from '../../services/dynamodb';
import { CustomersModule } from '../customers/customers.module';
import { CatalogModule } from '../catalog/catalog.module';
@Module({
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule],
controllers: [PlatformApiController],
providers: [PlatformApiService, DadosferaLogger],
exports: [PlatformApiService],
@@ -89,6 +89,12 @@ export class PlatformApiService {
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
this.logger.error('Platform API upstream error', {
status: response.status,
data: response.data,
path,
method: method.toUpperCase(),
});
throw new HttpException(response.data, response.status);
}
@@ -358,6 +358,81 @@ export class ElasticsearchService {
}
}
private getDataAssetIndex(customerName: string): string {
return `${customerName}_data_assets_catalog`;
}
async findDataAssetByTable(
customerName: string,
tableName: string,
tableSchema: string,
): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> {
const index = this.getDataAssetIndex(customerName);
this.logger.info('Elasticsearch: Searching data asset', {
index,
tableName,
tableSchema,
});
try {
const response = await this.client.post(`/${index}/_search`, {
query: {
bool: {
must: [
{ term: { 'table_name.keyword': tableName.toUpperCase() } },
{ term: { 'table_schema.keyword': tableSchema.toUpperCase() } },
],
},
},
size: 1,
});
const hits = response.data.hits?.hits || [];
if (hits.length === 0) {
this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index });
return null;
}
return { ...hits[0]._source, _es_id: hits[0]._id };
} catch (error) {
this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index });
throw error;
}
}
async updateDataAsset(
customerName: string,
assetId: string,
updates: Record<string, any>,
): Promise<any> {
const index = this.getDataAssetIndex(customerName);
this.logger.info('Elasticsearch: Updating data asset', {
index,
assetId,
fields: Object.keys(updates),
});
try {
const response = await this.client.post(
`/${index}/_update/${assetId}`,
{ doc: updates },
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Data asset updated', {
assetId,
result: response.data.result,
});
return response.data;
} catch (error) {
this.handleError('updateDataAsset', error, { assetId, index });
throw error;
}
}
private handleError(
operation: string,
error: any,