mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-21 21:49:09 +00:00
FEAT: Enable dataset registration via API for external sources
As part of our hybrid approach, we need to allow dataset cataloging from outside our cloud environment. Instead of relying on our GRPC APIs, we leveraged an existing REST API to handle this process. - Updated Maestro to call Nimbus directly for dataset registration. - Modified protospack to generate the necessary TypeScript interfaces. - Added the `/catalog/register-dataset` endpoint to the API. This change enables seamless dataset registration from external sources while maintaining compatibility with our existing infrastructure. Related tickets or issues: [TECN-9441]
This commit is contained in:
@@ -77,6 +77,8 @@ export class AuthCustomer {
|
||||
scheduleLimit: string;
|
||||
@ApiProperty()
|
||||
links: Link[];
|
||||
@ApiProperty()
|
||||
themeEnabled: boolean;
|
||||
}
|
||||
|
||||
export class AuthSignInReq implements AuthSignInRequest {
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
} from '@nestjs/microservices';
|
||||
import { credentials } from '@grpc/grpc-js';
|
||||
import { Catalog } from '@dadosfera/protospack-v2';
|
||||
import { PlatformInterfaces } from '@dadosfera/protospack-v2';
|
||||
|
||||
const isLocalConnection =
|
||||
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
||||
@@ -19,11 +20,13 @@ export class CatalogClientConfiguration {
|
||||
package: [
|
||||
Catalog.ProtoPackages.ReadPackage,
|
||||
Catalog.ProtoPackages.WritePackage,
|
||||
PlatformInterfaces.ProtoPackages.WritePackage
|
||||
],
|
||||
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
||||
protoPath: [
|
||||
Catalog.ProtoPaths.ReadFilePath,
|
||||
Catalog.ProtoPaths.WriteFilePath,
|
||||
PlatformInterfaces.ProtoPaths.WriteFilePath
|
||||
],
|
||||
loader: {
|
||||
keepCase: true,
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
Put,
|
||||
Query,
|
||||
UseFilters,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
@@ -30,6 +32,7 @@ import { CatalogService } from './catalog.service';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import {
|
||||
BatchRemoveRlsRulesRequest,
|
||||
GetDatasetCatalogTaskRes,
|
||||
@@ -54,6 +57,7 @@ import {
|
||||
AddRlsRuleRequest,
|
||||
GetNimbusDashboardsRequest,
|
||||
GetRlsRulesRequest,
|
||||
RegisterDatasetWithMetatadaRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||
|
||||
@ApiTags('Catalog')
|
||||
@@ -743,4 +747,105 @@ export class CatalogController {
|
||||
|
||||
return JSON.parse(dashboards);
|
||||
}
|
||||
|
||||
|
||||
@Post('register-dataset')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
|
||||
)
|
||||
async registerDatasetWithMetadataRequest(
|
||||
@Body() body: RegisterDatasetWithMetatadaRequest,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
try {
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
});
|
||||
|
||||
this.logger.log('Request from user', {
|
||||
user_id,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
// Create table metadata
|
||||
const tableMetadataBody = {
|
||||
table_metadata: body.table_metadata,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug('Creating table metadata', { body: tableMetadataBody });
|
||||
const table_metadata_id = await this.catalogService.createTableMetadata(
|
||||
tableMetadataBody,
|
||||
);
|
||||
this.logger.info(`table_metadata_id: ${table_metadata_id}`);
|
||||
|
||||
// Create column metadata
|
||||
const columnMetadataBody = {
|
||||
column_metadata: body.column_metadata,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug('Creating column metadata', {
|
||||
body: columnMetadataBody,
|
||||
});
|
||||
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
|
||||
|
||||
// Create data preview
|
||||
const dataPreviewBody = {
|
||||
data_preview: body.data_preview,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug('Creating data preview', { body: dataPreviewBody });
|
||||
const data_preview_id = await this.catalogService.createDataPreview(
|
||||
dataPreviewBody,
|
||||
);
|
||||
|
||||
// Catalog dataset item
|
||||
this.logger.debug('Cataloging dataset item', { table_metadata_id });
|
||||
|
||||
this.logger.info(`metadata: ${JSON.stringify(metadata)}`);
|
||||
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
|
||||
|
||||
this.logger.info('Dataset registration completed successfully');
|
||||
return {
|
||||
message: 'Dataset registered successfully',
|
||||
table_metadata_id: table_metadata_id,
|
||||
column_metadata_ids: column_metadata_ids,
|
||||
data_preview_id: data_preview_id,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to register dataset', {
|
||||
error: {
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
response: error.response?.data,
|
||||
status: error.response?.status,
|
||||
},
|
||||
context: {
|
||||
customer_name: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
table_metadata: body.table_metadata?.table_name,
|
||||
},
|
||||
});
|
||||
|
||||
throw new HttpException(
|
||||
{
|
||||
message: 'Ocorreu um erro ao registrar o dataset',
|
||||
error: error.message,
|
||||
code: 'REGISTRATION_FAILED',
|
||||
details: error.message,
|
||||
},
|
||||
HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,11 @@ import {
|
||||
ProtoServices,
|
||||
Messages,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
|
||||
import {
|
||||
Messages as PlatformInterfaceMessages,
|
||||
WriteService as PlatformInterfaceWriteService,
|
||||
ProtoServices as PlatformInterfacesProtoServices,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
|
||||
import {
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
@@ -33,6 +38,7 @@ import {
|
||||
class CatalogService implements OnModuleInit {
|
||||
catalogReadService: ReadService.CatalogReadServices;
|
||||
catalogWriteService: WriteService.CatalogWriteServices;
|
||||
platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
|
||||
logger: any;
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
@@ -54,9 +60,14 @@ class CatalogService implements OnModuleInit {
|
||||
this.grpcClient.getService<WriteService.CatalogWriteServices>(
|
||||
ProtoServices.CatalogWriteServices,
|
||||
);
|
||||
this.platformWriteService =
|
||||
this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
|
||||
PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
|
||||
);
|
||||
}
|
||||
|
||||
_getNimbusUrl(body) {
|
||||
this.logger.info(`Body: ${JSON.stringify(body)}`);
|
||||
const customer = body.info.customer.toLowerCase();
|
||||
|
||||
if (process.env.ENV === 'prd') {
|
||||
@@ -420,6 +431,123 @@ class CatalogService implements OnModuleInit {
|
||||
);
|
||||
return res.dashboards;
|
||||
}
|
||||
|
||||
async createTableMetadata(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
|
||||
|
||||
this.logger.info(`Creating table metadata`);
|
||||
this.logger.info(`Using endpoint: ${endpoint}`);
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body)}`);
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, body.table_metadata);
|
||||
|
||||
this.logger.info(
|
||||
`Table metadata created successfully with status ${status}`,
|
||||
);
|
||||
return data.id;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create table metadata failed with status ${
|
||||
error.response?.status
|
||||
} because of ${error.response?.data || error.message}`,
|
||||
{
|
||||
error: error.response?.data || error.message,
|
||||
status: error.response?.status,
|
||||
endpoint,
|
||||
payload: body.table_metadata,
|
||||
},
|
||||
);
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async createColumnMetadata(body: any): Promise<number[]> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
|
||||
|
||||
this.logger.info(`Creating column metadata`);
|
||||
this.logger.info(`Using endpoint: ${endpoint}`);
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body)}`);
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, body.column_metadata);
|
||||
|
||||
this.logger.info(
|
||||
`Column metadata created successfully with status ${status}`,
|
||||
);
|
||||
return data.map((column) => column.id);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create column metadata failed with status ${
|
||||
error.response?.status
|
||||
} because of ${error.response?.data || error.message}`,
|
||||
{
|
||||
error: error.response?.data || error.message,
|
||||
status: error.response?.status,
|
||||
endpoint,
|
||||
payload: body.column_metadata,
|
||||
},
|
||||
);
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async createDataPreview(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
|
||||
|
||||
this.logger.info(`Creating data preview`);
|
||||
this.logger.info(`Using endpoint: ${endpoint}`);
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body)}`);
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, body.data_preview);
|
||||
|
||||
this.logger.info(
|
||||
`Data preview created successfully with status ${status}`,
|
||||
);
|
||||
return data.id;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create data preview failed with status ${
|
||||
error.response?.status
|
||||
} because of ${error.response?.data || error.message}`,
|
||||
{
|
||||
error: error.response?.data || error.message,
|
||||
status: error.response?.status,
|
||||
endpoint,
|
||||
payload: body.data_preview,
|
||||
});
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||
const customer_name_raw = metadata.get('customer_name');
|
||||
this.logger.debug(`Raw customer_name value: ${JSON.stringify(customer_name_raw)}`);
|
||||
|
||||
const customer_name = customer_name_raw?.[0]?.toString();
|
||||
if (!customer_name) {
|
||||
throw new BadRequestException('Customer name not found in metadata');
|
||||
}
|
||||
const res = await lastValueFrom(
|
||||
this.platformWriteService.CatalogDataAssets(
|
||||
{
|
||||
data_assets: [
|
||||
{
|
||||
data_asset_id: table_metadata_id.toString(),
|
||||
customer_name: customer_name,
|
||||
data_asset_type: 'dataset',
|
||||
},
|
||||
],
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
export { CatalogService };
|
||||
|
||||
Reference in New Issue
Block a user