mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-06 12:19:08 +00:00
FIX: Calendar Fix
This commit is contained in:
@@ -3887,6 +3887,26 @@
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "catalog_date_from",
|
||||
"required": false,
|
||||
"in": "query",
|
||||
"description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
|
||||
"example": "2025-01-01",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "catalog_date_to",
|
||||
"required": false,
|
||||
"in": "query",
|
||||
"description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
|
||||
"example": "2025-12-31",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
@@ -4026,6 +4046,26 @@
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "catalog_date_from",
|
||||
"required": false,
|
||||
"in": "query",
|
||||
"description": "Data inicial para filtro de catálogo (formato: YYYY-MM-DD)",
|
||||
"example": "2025-01-01",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "catalog_date_to",
|
||||
"required": false,
|
||||
"in": "query",
|
||||
"description": "Data final para filtro de catálogo (formato: YYYY-MM-DD)",
|
||||
"example": "2025-12-31",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
import { TypeParser } from 'src/utils/FileParser/parser-types';
|
||||
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
|
||||
|
||||
|
||||
class CatalogService implements OnModuleInit {
|
||||
catalogReadService: ReadService.CatalogReadServices;
|
||||
catalogWriteService: WriteService.CatalogWriteServices;
|
||||
@@ -55,6 +56,7 @@ class CatalogService implements OnModuleInit {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
|
||||
onModuleInit() {
|
||||
this.catalogReadService =
|
||||
this.grpcClient.getService<ReadService.CatalogReadServices>(
|
||||
@@ -70,20 +72,24 @@ class CatalogService implements OnModuleInit {
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
_getNimbusUrl(body) {
|
||||
this.logger.debug(`Body: ${JSON.stringify(body)}`);
|
||||
const customer = body.info.customer.toLowerCase();
|
||||
|
||||
|
||||
if (process.env.ENV === 'prd') {
|
||||
return `https://nimbus-${customer}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
|
||||
return `https://nimbus-${customer}.${process.env.ENV.replace(
|
||||
'local',
|
||||
'stg',
|
||||
)}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
|
||||
async getPiiReporter(metadata: Metadata, type: TypeParser) {
|
||||
this.logger.info('getPiiReporter: ' + type)
|
||||
try {
|
||||
@@ -94,8 +100,10 @@ class CatalogService implements OnModuleInit {
|
||||
)
|
||||
this.logger.info("Finish grpc call")
|
||||
|
||||
|
||||
const parser = ParserBuilder.build<PiiMetadata>(type);
|
||||
|
||||
|
||||
this.logger.info('parser file to: ' + type)
|
||||
const file = await parser.parse(data)
|
||||
this.logger.info('finish parser')
|
||||
@@ -105,9 +113,11 @@ class CatalogService implements OnModuleInit {
|
||||
'pdf': 'application/pdf'
|
||||
}
|
||||
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const filename = `relatorio-pii-${timestamp}.${type}`;
|
||||
|
||||
|
||||
return {
|
||||
file,
|
||||
filename: filename,
|
||||
@@ -118,20 +128,25 @@ class CatalogService implements OnModuleInit {
|
||||
throw error;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
if (!data.embed) data.embed = undefined;
|
||||
|
||||
|
||||
return lastValueFrom(
|
||||
this.catalogWriteService.CreateDataAsset(data, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
|
||||
|
||||
return lastValueFrom(
|
||||
this.catalogWriteService.ManagePermission(data, metadata),
|
||||
).catch((err) => {
|
||||
@@ -142,9 +157,11 @@ class CatalogService implements OnModuleInit {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async revokePermissions(data: Messages.RevokePermissionRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
|
||||
|
||||
return lastValueFrom(
|
||||
this.catalogWriteService.RevokePermission(data, metadata),
|
||||
).catch((err) => {
|
||||
@@ -155,9 +172,11 @@ class CatalogService implements OnModuleInit {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async commentOnDataAsset(data: Messages.MakeACommentRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
|
||||
|
||||
return lastValueFrom(
|
||||
this.catalogWriteService.MakeAComment(data, metadata),
|
||||
).catch((err) => {
|
||||
@@ -168,9 +187,11 @@ class CatalogService implements OnModuleInit {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async deleteComment(data: Messages.UpdateACommentRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
|
||||
|
||||
return lastValueFrom(
|
||||
this.catalogWriteService.UpdateAComment(data, metadata),
|
||||
).catch((err) => {
|
||||
@@ -181,9 +202,11 @@ class CatalogService implements OnModuleInit {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async deleteDataAsset(data: Messages.DeleteDataAssetRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
|
||||
|
||||
return lastValueFrom(
|
||||
this.catalogWriteService.DeleteDataAsset(data, metadata),
|
||||
).catch((err) => {
|
||||
@@ -194,16 +217,20 @@ class CatalogService implements OnModuleInit {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async getUserRolesIds(userId: string) {
|
||||
const result = await this.userService.findOneById(userId).catch(() => null);
|
||||
|
||||
|
||||
if (result) {
|
||||
return result.user.roles.map((role) => role.id);
|
||||
}
|
||||
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
|
||||
async searchDataAssets(
|
||||
query: Record<string, any>,
|
||||
metadata: Metadata,
|
||||
@@ -211,10 +238,19 @@ class CatalogService implements OnModuleInit {
|
||||
) {
|
||||
this.logger.info('CatalogService - searchDataAssets', { query });
|
||||
|
||||
|
||||
const { search, page, size, sort_by, order, ...filters } = query;
|
||||
|
||||
|
||||
this.logger.debug('Extracted filters:', { filters });
|
||||
|
||||
|
||||
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM:', {
|
||||
search, page, size, sort_by, order,
|
||||
filters: filters
|
||||
});
|
||||
|
||||
|
||||
if (filters.owner) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
@@ -224,6 +260,7 @@ class CatalogService implements OnModuleInit {
|
||||
users: customer_users.slice(0, 5).map(u => ({ id: u.id, email: u.email, name: u.name }))
|
||||
});
|
||||
|
||||
|
||||
const ownerValues = Array.isArray(filters.owner)
|
||||
? filters.owner
|
||||
: (typeof filters.owner === 'string' && filters.owner.includes(',')
|
||||
@@ -232,6 +269,7 @@ class CatalogService implements OnModuleInit {
|
||||
|
||||
this.logger.info('Owner values to convert:', { ownerValues, ownerFiltersOriginal: filters.owner });
|
||||
|
||||
|
||||
const ownerIds = ownerValues
|
||||
.map((ownerValue: string) => {
|
||||
const normalizedOwner = ownerValue.replace(/\s/g, '+');
|
||||
@@ -256,11 +294,13 @@ class CatalogService implements OnModuleInit {
|
||||
})
|
||||
.filter((id: string) => id);
|
||||
|
||||
|
||||
if (ownerIds.length > 0) {
|
||||
filters.owner = ownerIds;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const { data_assets, total } = await lastValueFrom(
|
||||
this.catalogReadService.GetAllDataAssets(
|
||||
{
|
||||
@@ -275,16 +315,25 @@ class CatalogService implements OnModuleInit {
|
||||
),
|
||||
);
|
||||
|
||||
|
||||
|
||||
console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
|
||||
|
||||
|
||||
|
||||
const result = JSON.parse(data_assets);
|
||||
|
||||
|
||||
const response = await this.getAssetsUsersAndRoles(
|
||||
result.data_assets,
|
||||
customer_id,
|
||||
);
|
||||
|
||||
|
||||
return { data_assets: response, total };
|
||||
}
|
||||
|
||||
|
||||
async downloadAssets(
|
||||
query: Record<string, any>,
|
||||
metadata: Metadata,
|
||||
@@ -292,6 +341,7 @@ class CatalogService implements OnModuleInit {
|
||||
) {
|
||||
const data = await this.searchDataAssets(query, metadata, customer_id);
|
||||
|
||||
|
||||
const formatData = data.data_assets.map(asset => ({
|
||||
id: asset.id,
|
||||
display_name: asset.display_name,
|
||||
@@ -300,19 +350,24 @@ class CatalogService implements OnModuleInit {
|
||||
tags: '[' + asset.tags.join(', ') + ']'
|
||||
}))
|
||||
|
||||
|
||||
const parser = ParserBuilder.build<AssetReporter>('csv');
|
||||
|
||||
|
||||
const file = await parser.parse(formatData);
|
||||
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const filename = `dadosfera_assets_${timestamp}.csv`;
|
||||
|
||||
|
||||
return {
|
||||
file,
|
||||
filename
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async getOneDataAsset(data: {
|
||||
id: string;
|
||||
customer_id: string;
|
||||
@@ -333,9 +388,11 @@ class CatalogService implements OnModuleInit {
|
||||
};
|
||||
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
||||
|
||||
|
||||
return { data_asset: asset[0] };
|
||||
}
|
||||
|
||||
|
||||
async getOneDataAssetByPipelineAndObject(data: {
|
||||
customer_id: string;
|
||||
pipeline: string;
|
||||
@@ -357,9 +414,11 @@ class CatalogService implements OnModuleInit {
|
||||
};
|
||||
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
||||
|
||||
|
||||
return { data_asset: asset[0] };
|
||||
}
|
||||
|
||||
|
||||
async updateOneDataAsset(data: {
|
||||
data_asset_id: string;
|
||||
customer_id: string;
|
||||
@@ -368,6 +427,7 @@ class CatalogService implements OnModuleInit {
|
||||
}) {
|
||||
const { body, customer_id, data_asset_id, metadata } = data;
|
||||
|
||||
|
||||
const { data_asset } = await lastValueFrom(
|
||||
this.catalogWriteService.UpdateDataAsset(
|
||||
{ id: data_asset_id, changes: JSON.stringify(body) },
|
||||
@@ -382,9 +442,11 @@ class CatalogService implements OnModuleInit {
|
||||
};
|
||||
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
||||
|
||||
|
||||
return { data_asset: asset[0] };
|
||||
}
|
||||
|
||||
|
||||
async getDataDocs(id: string, metadata: Metadata) {
|
||||
const { documentation } = await lastValueFrom(
|
||||
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
|
||||
@@ -394,6 +456,7 @@ class CatalogService implements OnModuleInit {
|
||||
return docs;
|
||||
}
|
||||
|
||||
|
||||
async getDatasetPreview(id: string, metadata: Metadata) {
|
||||
const { preview } = await lastValueFrom(
|
||||
this.catalogReadService.GetDatasetPreview(
|
||||
@@ -405,6 +468,7 @@ class CatalogService implements OnModuleInit {
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
|
||||
const { columns_metadata } = await lastValueFrom(
|
||||
this.catalogReadService.GetDatasetColumnsMetadata(
|
||||
@@ -416,6 +480,7 @@ class CatalogService implements OnModuleInit {
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
async createDataDocs(body) {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
const { data } = await axios.post(
|
||||
@@ -425,9 +490,11 @@ class CatalogService implements OnModuleInit {
|
||||
return data;
|
||||
}
|
||||
|
||||
|
||||
async findAllTags(data, metadata) {
|
||||
this.logger.info('CatalogService - findAllCustomerTags');
|
||||
|
||||
|
||||
const response = await lastValueFrom(
|
||||
this.catalogReadService.GetCustomerTags(data, metadata),
|
||||
)
|
||||
@@ -440,6 +507,7 @@ class CatalogService implements OnModuleInit {
|
||||
throw new Error(err);
|
||||
});
|
||||
|
||||
|
||||
return response;
|
||||
}
|
||||
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
|
||||
@@ -454,6 +522,7 @@ class CatalogService implements OnModuleInit {
|
||||
(u) => u.id === data_asset.owner,
|
||||
)?.email;
|
||||
|
||||
|
||||
const roles = [];
|
||||
const users = [];
|
||||
const data_asset_roles = data_asset?.roles || []
|
||||
@@ -462,6 +531,7 @@ class CatalogService implements OnModuleInit {
|
||||
if (role) roles.push({ id: role.id, name: role.name });
|
||||
}
|
||||
|
||||
|
||||
const data_asset_users = data_asset?.users || []
|
||||
for (const user_id of data_asset_users) {
|
||||
const user = customer_users.find((r) => r.id === user_id);
|
||||
@@ -476,6 +546,7 @@ class CatalogService implements OnModuleInit {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
|
||||
const { session } = await lastValueFrom(
|
||||
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
|
||||
@@ -489,6 +560,7 @@ class CatalogService implements OnModuleInit {
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
|
||||
const res = await lastValueFrom(
|
||||
this.catalogWriteService.AddRlsRule(data, metadata),
|
||||
@@ -496,6 +568,7 @@ class CatalogService implements OnModuleInit {
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
async removeRlsRule(id: number, metadata: Metadata) {
|
||||
const res = await lastValueFrom(
|
||||
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
|
||||
@@ -503,12 +576,14 @@ class CatalogService implements OnModuleInit {
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
async batchRemoveRlsRule(
|
||||
query: BatchRemoveRlsRulesRequest,
|
||||
metadata: Metadata,
|
||||
) {
|
||||
const { id_rls, nimbus_dashboard_id } = query;
|
||||
|
||||
|
||||
if (id_rls && nimbus_dashboard_id) {
|
||||
throw new BadRequestException(
|
||||
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
|
||||
@@ -529,6 +604,7 @@ class CatalogService implements OnModuleInit {
|
||||
return 'OK';
|
||||
}
|
||||
|
||||
|
||||
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
|
||||
const res = await lastValueFrom(
|
||||
this.catalogReadService.GetRlsRules(data, metadata),
|
||||
@@ -536,6 +612,7 @@ class CatalogService implements OnModuleInit {
|
||||
return res.rls_rules;
|
||||
}
|
||||
|
||||
|
||||
async getOneRlsRule(id: number, metadata: Metadata) {
|
||||
const res = await lastValueFrom(
|
||||
this.catalogReadService.GetOneRlsRule({ id }, metadata),
|
||||
@@ -543,6 +620,7 @@ class CatalogService implements OnModuleInit {
|
||||
return res.rls_rule;
|
||||
}
|
||||
|
||||
|
||||
async getNimbusDashboards(
|
||||
data: GetNimbusDashboardsRequest,
|
||||
metadata: Metadata,
|
||||
@@ -553,19 +631,24 @@ class CatalogService implements OnModuleInit {
|
||||
return res.dashboards;
|
||||
}
|
||||
|
||||
|
||||
async createTableMetadata(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||
|
||||
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
|
||||
|
||||
|
||||
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
|
||||
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
|
||||
|
||||
|
||||
this.logger.info(
|
||||
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
|
||||
{...body.logMetadata},
|
||||
@@ -580,19 +663,23 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async createColumnMetadata(body: any): Promise<number[]> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
|
||||
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
try {
|
||||
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
|
||||
const { data, status } = await axios.post(endpoint, body.column_metadata);
|
||||
|
||||
|
||||
this.logger.info(
|
||||
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
|
||||
{...body.logMetadata},
|
||||
@@ -607,18 +694,22 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async createDataPreview(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
|
||||
|
||||
|
||||
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
|
||||
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, body.data_preview);
|
||||
|
||||
|
||||
this.logger.info(
|
||||
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
|
||||
{...body.logMetadata},
|
||||
@@ -635,9 +726,11 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||
const customer_name_raw = metadata.get('customer_name');
|
||||
|
||||
|
||||
const customer_name = customer_name_raw?.[0]?.toString();
|
||||
if (!customer_name) {
|
||||
throw new BadRequestException('Customer name not found in metadata');
|
||||
@@ -660,4 +753,5 @@ class CatalogService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
export { CatalogService };
|
||||
|
||||
export { CatalogService };
|
||||
|
||||
@@ -153,6 +153,18 @@ export class ICatalogAllRequest {
|
||||
example: 'user-id-1,user-id-2',
|
||||
})
|
||||
owner?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)',
|
||||
example: '2025-01-01',
|
||||
})
|
||||
catalog_date_from?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)',
|
||||
example: '2025-12-31',
|
||||
})
|
||||
catalog_date_to?: string;
|
||||
}
|
||||
|
||||
export class ICatalogAllResponse {
|
||||
|
||||
Reference in New Issue
Block a user