mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-21 03:59:08 +00:00
feat: add rename-tables endpoint with catalog sync and rollback
Add POST /platform/jobs/:jobId/rename-tables that renames Snowflake tables via platform-api and syncs the rename to Elasticsearch and Nimbus (table-metadata, column-metadata, data-preview). If catalog sync fails, all completed catalog steps are rolled back in reverse order and the Snowflake rename is reverted. - Support any connector type (jdbc, singer, s3) via getJobByAnyConnectorType - Resolve old table names from output_config (raw/qualify) - Skip qualify sync when output_config.qualify has no table_name - Add findDataAssetByPipelineAndTable and updateDataAsset to ElasticsearchService - Add renameTableOnNimbus, renameColumnMetadataOnNimbus, renameDataPreviewOnNimbus to CatalogService - Add upstream error logging to PlatformApiService Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
1a2153d62f
commit
38a9e21f5f
@@ -734,6 +734,64 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
async renameTableOnNimbus(
|
||||
nimbusUrl: string,
|
||||
nimbusId: number,
|
||||
changes: { table_name?: string; table_schema?: string; display_name?: string },
|
||||
): Promise<void> {
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
|
||||
this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
|
||||
await axios.patch(endpoint, changes);
|
||||
}
|
||||
|
||||
async renameColumnMetadataOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
|
||||
const { data: columns } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(columns) ? columns : [];
|
||||
|
||||
for (const column of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
|
||||
}
|
||||
|
||||
async renameDataPreviewOnNimbus(
|
||||
nimbusUrl: string,
|
||||
databaseName: string,
|
||||
oldTableName: string,
|
||||
oldTableSchema: string,
|
||||
newTableName: string,
|
||||
newTableSchema: string,
|
||||
): Promise<void> {
|
||||
const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
|
||||
this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
|
||||
const { data: previews } = await axios.get(listEndpoint);
|
||||
|
||||
const filtered = Array.isArray(previews) ? previews : [];
|
||||
|
||||
for (const preview of filtered) {
|
||||
const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
|
||||
await axios.patch(patchEndpoint, {
|
||||
table_name: newTableName,
|
||||
table_schema: newTableSchema,
|
||||
});
|
||||
}
|
||||
this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
|
||||
}
|
||||
|
||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||
const customer_name_raw = metadata.get('customer_name');
|
||||
|
||||
|
||||
Reference in New Issue
Block a user