FEAT: Enable dataset registration via API for external sources

As part of our hybrid approach, we need to allow dataset cataloging from outside our cloud environment.
Instead of relying on our GRPC APIs, we leveraged an existing REST API to handle this process.

- Updated Maestro to call Nimbus directly for dataset registration.
- Modified protospack to generate the necessary TypeScript interfaces.
- Added the `/catalog/register-dataset` endpoint to the API.

This change enables seamless dataset registration from external sources while maintaining compatibility with our existing infrastructure.

Related tickets or issues: [TECN-9441]
This commit is contained in:
Rafael
2025-02-14 10:08:45 -03:00
parent e316bd1a7b
commit 721232448e
7 changed files with 1900 additions and 1407 deletions
+2
View File
@@ -77,6 +77,8 @@ export class AuthCustomer {
scheduleLimit: string;
@ApiProperty()
links: Link[];
@ApiProperty()
themeEnabled: boolean;
}
export class AuthSignInReq implements AuthSignInRequest {
+3
View File
@@ -5,6 +5,7 @@ import {
} from '@nestjs/microservices';
import { credentials } from '@grpc/grpc-js';
import { Catalog } from '@dadosfera/protospack-v2';
import { PlatformInterfaces } from '@dadosfera/protospack-v2';
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
@@ -19,11 +20,13 @@ export class CatalogClientConfiguration {
package: [
Catalog.ProtoPackages.ReadPackage,
Catalog.ProtoPackages.WritePackage,
PlatformInterfaces.ProtoPackages.WritePackage
],
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: [
Catalog.ProtoPaths.ReadFilePath,
Catalog.ProtoPaths.WriteFilePath,
PlatformInterfaces.ProtoPaths.WriteFilePath
],
loader: {
keepCase: true,
+105
View File
@@ -13,6 +13,8 @@ import {
Put,
Query,
UseFilters,
HttpException,
HttpStatus,
} from '@nestjs/common';
import {
ApiCreatedResponse,
@@ -30,6 +32,7 @@ import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { lastValueFrom } from 'rxjs';
import {
BatchRemoveRlsRulesRequest,
GetDatasetCatalogTaskRes,
@@ -54,6 +57,7 @@ import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
@ApiTags('Catalog')
@@ -743,4 +747,105 @@ export class CatalogController {
return JSON.parse(dashboards);
}
@Post('register-dataset')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
)
async registerDatasetWithMetadataRequest(
@Body() body: RegisterDatasetWithMetatadaRequest,
@User() user: RequestUser,
) {
try {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
this.logger.log('Request from user', {
user_id,
customer_name,
});
// Create table metadata
const tableMetadataBody = {
table_metadata: body.table_metadata,
info: {
customer: customer_name,
},
};
this.logger.debug('Creating table metadata', { body: tableMetadataBody });
const table_metadata_id = await this.catalogService.createTableMetadata(
tableMetadataBody,
);
this.logger.info(`table_metadata_id: ${table_metadata_id}`);
// Create column metadata
const columnMetadataBody = {
column_metadata: body.column_metadata,
info: {
customer: customer_name,
},
};
this.logger.debug('Creating column metadata', {
body: columnMetadataBody,
});
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
// Create data preview
const dataPreviewBody = {
data_preview: body.data_preview,
info: {
customer: customer_name,
},
};
this.logger.debug('Creating data preview', { body: dataPreviewBody });
const data_preview_id = await this.catalogService.createDataPreview(
dataPreviewBody,
);
// Catalog dataset item
this.logger.debug('Cataloging dataset item', { table_metadata_id });
this.logger.info(`metadata: ${JSON.stringify(metadata)}`);
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
this.logger.info('Dataset registration completed successfully');
return {
message: 'Dataset registered successfully',
table_metadata_id: table_metadata_id,
column_metadata_ids: column_metadata_ids,
data_preview_id: data_preview_id,
};
} catch (error) {
this.logger.error('Failed to register dataset', {
error: {
message: error.message,
stack: error.stack,
response: error.response?.data,
status: error.response?.status,
},
context: {
customer_name: user.customer_name,
user_id: user.user_id,
table_metadata: body.table_metadata?.table_name,
},
});
throw new HttpException(
{
message: 'Ocorreu um erro ao registrar o dataset',
error: error.message,
code: 'REGISTRATION_FAILED',
details: error.message,
},
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
}
+128
View File
@@ -5,6 +5,11 @@ import {
ProtoServices,
Messages,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
Messages as PlatformInterfaceMessages,
WriteService as PlatformInterfaceWriteService,
ProtoServices as PlatformInterfacesProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
import {
BadRequestException,
HttpException,
@@ -33,6 +38,7 @@ import {
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
catalogWriteService: WriteService.CatalogWriteServices;
platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
logger: any;
constructor(
@Inject(DadosferaLogger)
@@ -54,9 +60,14 @@ class CatalogService implements OnModuleInit {
this.grpcClient.getService<WriteService.CatalogWriteServices>(
ProtoServices.CatalogWriteServices,
);
this.platformWriteService =
this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
);
}
_getNimbusUrl(body) {
this.logger.info(`Body: ${JSON.stringify(body)}`);
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
@@ -420,6 +431,123 @@ class CatalogService implements OnModuleInit {
);
return res.dashboards;
}
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata`);
this.logger.info(`Using endpoint: ${endpoint}`);
this.logger.debug(`Payload: ${JSON.stringify(body)}`);
try {
const { data, status } = await axios.post(endpoint, body.table_metadata);
this.logger.info(
`Table metadata created successfully with status ${status}`,
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{
error: error.response?.data || error.message,
status: error.response?.status,
endpoint,
payload: body.table_metadata,
},
);
throw new Error(error.response?.data?.message || error.message);
}
}
async createColumnMetadata(body: any): Promise<number[]> {
const nimbusUrl = this._getNimbusUrl(body);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
this.logger.info(`Creating column metadata`);
this.logger.info(`Using endpoint: ${endpoint}`);
this.logger.debug(`Payload: ${JSON.stringify(body)}`);
try {
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Column metadata created successfully with status ${status}`,
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{
error: error.response?.data || error.message,
status: error.response?.status,
endpoint,
payload: body.column_metadata,
},
);
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview`);
this.logger.info(`Using endpoint: ${endpoint}`);
this.logger.debug(`Payload: ${JSON.stringify(body)}`);
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status}`,
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create data preview failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{
error: error.response?.data || error.message,
status: error.response?.status,
endpoint,
payload: body.data_preview,
});
throw new Error(error.response?.data?.message || error.message);
}
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
this.logger.debug(`Raw customer_name value: ${JSON.stringify(customer_name_raw)}`);
const customer_name = customer_name_raw?.[0]?.toString();
if (!customer_name) {
throw new BadRequestException('Customer name not found in metadata');
}
const res = await lastValueFrom(
this.platformWriteService.CatalogDataAssets(
{
data_assets: [
{
data_asset_id: table_metadata_id.toString(),
customer_name: customer_name,
data_asset_type: 'dataset',
},
],
},
metadata,
),
);
return res;
}
}
export { CatalogService };