mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-22 11:49:07 +00:00
FEAT: Enable dataset registration via API for external sources
As part of our hybrid approach, we need to allow dataset cataloging from outside our cloud environment. Instead of relying on our GRPC APIs, we leveraged an existing REST API to handle this process. - Updated Maestro to call Nimbus directly for dataset registration. - Modified protospack to generate the necessary TypeScript interfaces. - Added the `/catalog/register-dataset` endpoint to the API. This change enables seamless dataset registration from external sources while maintaining compatibility with our existing infrastructure. Related tickets or issues: [TECN-9441]
This commit is contained in:
@@ -13,6 +13,8 @@ import {
|
||||
Put,
|
||||
Query,
|
||||
UseFilters,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
@@ -30,6 +32,7 @@ import { CatalogService } from './catalog.service';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import {
|
||||
BatchRemoveRlsRulesRequest,
|
||||
GetDatasetCatalogTaskRes,
|
||||
@@ -54,6 +57,7 @@ import {
|
||||
AddRlsRuleRequest,
|
||||
GetNimbusDashboardsRequest,
|
||||
GetRlsRulesRequest,
|
||||
RegisterDatasetWithMetatadaRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||
|
||||
@ApiTags('Catalog')
|
||||
@@ -743,4 +747,105 @@ export class CatalogController {
|
||||
|
||||
return JSON.parse(dashboards);
|
||||
}
|
||||
|
||||
|
||||
@Post('register-dataset')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
|
||||
)
|
||||
async registerDatasetWithMetadataRequest(
|
||||
@Body() body: RegisterDatasetWithMetatadaRequest,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
try {
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
});
|
||||
|
||||
this.logger.log('Request from user', {
|
||||
user_id,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
// Create table metadata
|
||||
const tableMetadataBody = {
|
||||
table_metadata: body.table_metadata,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug('Creating table metadata', { body: tableMetadataBody });
|
||||
const table_metadata_id = await this.catalogService.createTableMetadata(
|
||||
tableMetadataBody,
|
||||
);
|
||||
this.logger.info(`table_metadata_id: ${table_metadata_id}`);
|
||||
|
||||
// Create column metadata
|
||||
const columnMetadataBody = {
|
||||
column_metadata: body.column_metadata,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug('Creating column metadata', {
|
||||
body: columnMetadataBody,
|
||||
});
|
||||
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
|
||||
|
||||
// Create data preview
|
||||
const dataPreviewBody = {
|
||||
data_preview: body.data_preview,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug('Creating data preview', { body: dataPreviewBody });
|
||||
const data_preview_id = await this.catalogService.createDataPreview(
|
||||
dataPreviewBody,
|
||||
);
|
||||
|
||||
// Catalog dataset item
|
||||
this.logger.debug('Cataloging dataset item', { table_metadata_id });
|
||||
|
||||
this.logger.info(`metadata: ${JSON.stringify(metadata)}`);
|
||||
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
|
||||
|
||||
this.logger.info('Dataset registration completed successfully');
|
||||
return {
|
||||
message: 'Dataset registered successfully',
|
||||
table_metadata_id: table_metadata_id,
|
||||
column_metadata_ids: column_metadata_ids,
|
||||
data_preview_id: data_preview_id,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to register dataset', {
|
||||
error: {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
response: error.response?.data,
|
||||
status: error.response?.status,
|
||||
},
|
||||
context: {
|
||||
customer_name: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
table_metadata: body.table_metadata?.table_name,
|
||||
},
|
||||
});
|
||||
|
||||
throw new HttpException(
|
||||
{
|
||||
message: 'Ocorreu um erro ao registrar o dataset',
|
||||
error: error.message,
|
||||
code: 'REGISTRATION_FAILED',
|
||||
details: error.message,
|
||||
},
|
||||
HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user