From dc617b6925c10d3aedd08d1db6ff385ab2f31bab Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 14 Feb 2025 15:04:22 -0300 Subject: [PATCH] REFACTOR: Improve logging with additional metadata - Enhanced logs to include more detailed request body information. - Standardized log calls to include extra metadata, such as service name, environment, customer name, user ID, HTTP method, and request path. - Improved error logging to provide better debugging insights. Example log format: EXTRA[{"service":{"name":"maestro","environment":"stg"},"customer_name":"dadosfera","user_id":"047824d8-0021-709a-bf15-39b8dd069aaf","method":"POST","path":"/catalog/register-dataset"}] Related tickets or issues: [TECN-9441] --- src/modules/catalog/catalog.controller.ts | 47 ++++++--------- src/modules/catalog/catalog.service.ts | 69 ++++++++++------------- 2 files changed, 47 insertions(+), 69 deletions(-) diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 9c366bb..5f2d194 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -758,8 +758,15 @@ export class CatalogController { @Body() body: RegisterDatasetWithMetatadaRequest, @User() user: RequestUser, ) { - try { const { customer_id, customer_name, user_id, username } = user; + const logMetadata = { + customer_name: customer_name, + user_id: user_id, + method: 'POST', + path: '/catalog/register-dataset', + }; + try { + const metadata = PackTheMetadata({ customer_id, customer_name, @@ -767,23 +774,22 @@ export class CatalogController { username, }); - this.logger.log('Request from user', { - user_id, - customer_name, - }); + this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata); // Create table metadata const tableMetadataBody = { table_metadata: body.table_metadata, info: { customer: customer_name, + }, + logMetadata: logMetadata }; - this.logger.debug('Creating table metadata', { body: tableMetadataBody }); + const table_metadata_id = await this.catalogService.createTableMetadata( tableMetadataBody, ); - this.logger.info(`table_metadata_id: ${table_metadata_id}`); + this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata); // Create column metadata const columnMetadataBody = { @@ -792,9 +798,7 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.debug('Creating column metadata', { - body: columnMetadataBody, - }); + this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata); const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody) // Create data preview @@ -804,18 +808,15 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.debug('Creating data preview', { body: dataPreviewBody }); + this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata); const data_preview_id = await this.catalogService.createDataPreview( dataPreviewBody, ); // Catalog dataset item - this.logger.debug('Cataloging dataset item', { table_metadata_id }); - - this.logger.info(`metadata: ${JSON.stringify(metadata)}`); + this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata); await this.catalogService.catalogDatasetItem(table_metadata_id, metadata); - - this.logger.info('Dataset registration completed successfully'); + this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata); return { message: 'Dataset registered successfully', table_metadata_id: table_metadata_id, @@ -823,19 +824,7 @@ export class CatalogController { data_preview_id: data_preview_id, }; } catch (error) { - this.logger.error('Failed to register dataset', { - error: { - message: error.message, - stack: error.stack, - response: error.response?.data, - status: error.response?.status, - }, - context: { - customer_name: user.customer_name, - user_id: user.user_id, - table_metadata: body.table_metadata?.table_name, - }, - }); + this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata); throw new HttpException( { diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 5101a25..ceb8205 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -67,7 +67,7 @@ class CatalogService implements OnModuleInit { } _getNimbusUrl(body) { - this.logger.info(`Body: ${JSON.stringify(body)}`); + this.logger.debug(`Body: ${JSON.stringify(body)}`); const customer = body.info.customer.toLowerCase(); if (process.env.ENV === 'prd') { @@ -434,99 +434,88 @@ class CatalogService implements OnModuleInit { async createTableMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); + const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`; - this.logger.info(`Creating table metadata`); - this.logger.info(`Using endpoint: ${endpoint}`); - this.logger.debug(`Payload: ${JSON.stringify(body)}`); + this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata}); try { - const { data, status } = await axios.post(endpoint, body.table_metadata); + const { data, status } = await axios.post(endpoint, {...body.table_metadata}); this.logger.info( - `Table metadata created successfully with status ${status}`, + `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`, + {...body.logMetadata}, ); return data.id; } catch (error) { this.logger.error( - `Failed to create table metadata failed with status ${ + `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${ error.response?.status - } because of ${error.response?.data || error.message}`, - { - error: error.response?.data || error.message, - status: error.response?.status, - endpoint, - payload: body.table_metadata, - }, - ); + } because of ${error.response?.data || error.message}`, {...body.logMetadata}); throw new Error(error.response?.data?.message || error.message); } } async createColumnMetadata(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata); const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`; - this.logger.info(`Creating column metadata`); - this.logger.info(`Using endpoint: ${endpoint}`); - this.logger.debug(`Payload: ${JSON.stringify(body)}`); + try { + this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata}); const { data, status } = await axios.post(endpoint, body.column_metadata); this.logger.info( - `Column metadata created successfully with status ${status}`, + `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`, + {...body.logMetadata}, ); return data.map((column) => column.id); } catch (error) { this.logger.error( - `Failed to create column metadata failed with status ${ + `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${ error.response?.status - } because of ${error.response?.data || error.message}`, - { - error: error.response?.data || error.message, - status: error.response?.status, - endpoint, - payload: body.column_metadata, - }, - ); + } because of ${error.response?.data || error.message}`, {...body.logMetadata}); throw new Error(error.response?.data?.message || error.message); } } async createDataPreview(body: any): Promise { const nimbusUrl = this._getNimbusUrl(body); + this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata}); const endpoint = `${nimbusUrl}/api/catalog/data-preview/`; - this.logger.info(`Creating data preview`); - this.logger.info(`Using endpoint: ${endpoint}`); - this.logger.debug(`Payload: ${JSON.stringify(body)}`); + this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata}); + this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata}); + this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata}); try { const { data, status } = await axios.post(endpoint, body.data_preview); this.logger.info( - `Data preview created successfully with status ${status}`, + `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`, + {...body.logMetadata}, ); return data.id; } catch (error) { this.logger.error( - `Failed to create data preview failed with status ${ + `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${ error.response?.status } because of ${error.response?.data || error.message}`, - { - error: error.response?.data || error.message, - status: error.response?.status, - endpoint, - payload: body.data_preview, - }); + {...body.logMetadata}, + ); throw new Error(error.response?.data?.message || error.message); } } async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) { const customer_name_raw = metadata.get('customer_name'); - this.logger.debug(`Raw customer_name value: ${JSON.stringify(customer_name_raw)}`); const customer_name = customer_name_raw?.[0]?.toString(); if (!customer_name) {