REFACTOR: Improve logging with additional metadata

- Enhanced logs to include more detailed request body information.
- Standardized log calls to include extra metadata, such as service name, environment, customer name, user ID, HTTP method, and request path.
- Improved error logging to provide better debugging insights.

Example log format:
EXTRA[{"service":{"name":"maestro","environment":"stg"},"customer_name":"dadosfera","user_id":"047824d8-0021-709a-bf15-39b8dd069aaf","method":"POST","path":"/catalog/register-dataset"}]

Related tickets or issues: [TECN-9441]

src/modules/catalog/catalog.controller.ts

@@ -758,8 +758,15 @@ export class CatalogController {
     @Body() body: RegisterDatasetWithMetatadaRequest,
     @User() user: RequestUser,
   ) {
-    try {
       const { customer_id, customer_name, user_id, username } = user;
+      const logMetadata = {
+        customer_name: customer_name,
+        user_id: user_id,
+        method: 'POST',
+        path: '/catalog/register-dataset',
+      };
+    try {
+
       const metadata = PackTheMetadata({
         customer_id,
         customer_name,
@@ -767,23 +774,22 @@ export class CatalogController {
         username,
       });
 
-      this.logger.log('Request from user', {
-        user_id,
-        customer_name,
-      });
+      this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
 
       // Create table metadata
       const tableMetadataBody = {
         table_metadata: body.table_metadata,
         info: {
           customer: customer_name,
+          
         },
+        logMetadata: logMetadata
       };
-      this.logger.debug('Creating table metadata', { body: tableMetadataBody });
+      
       const table_metadata_id = await this.catalogService.createTableMetadata(
         tableMetadataBody,
       );
-      this.logger.info(`table_metadata_id: ${table_metadata_id}`);
+      this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
 
       // Create column metadata
       const columnMetadataBody = {
@@ -792,9 +798,7 @@ export class CatalogController {
           customer: customer_name,
         },
       };
-      this.logger.debug('Creating column metadata', {
-        body: columnMetadataBody,
-      });
+      this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
       const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
 
       // Create data preview
@@ -804,18 +808,15 @@ export class CatalogController {
           customer: customer_name,
         },
       };
-      this.logger.debug('Creating data preview', { body: dataPreviewBody });
+      this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
       const data_preview_id = await this.catalogService.createDataPreview(
         dataPreviewBody,
       );
 
       // Catalog dataset item
-      this.logger.debug('Cataloging dataset item', { table_metadata_id });
-
-      this.logger.info(`metadata: ${JSON.stringify(metadata)}`);
+      this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
       await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
-
-      this.logger.info('Dataset registration completed successfully');
+      this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
       return {
         message: 'Dataset registered successfully',
         table_metadata_id: table_metadata_id,
@@ -823,19 +824,7 @@ export class CatalogController {
         data_preview_id: data_preview_id,
       };
     } catch (error) {
-      this.logger.error('Failed to register dataset', {
-        error: {
-          message: error.message,
-          stack: error.stack,
-          response: error.response?.data,
-          status: error.response?.status,
-        },
-        context: {
-          customer_name: user.customer_name,
-          user_id: user.user_id,
-          table_metadata: body.table_metadata?.table_name,
-        },
-      });
+      this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
 
       throw new HttpException(
         {

src/modules/catalog/catalog.service.ts

@@ -67,7 +67,7 @@ class CatalogService implements OnModuleInit {
   }
 
   _getNimbusUrl(body) {
-    this.logger.info(`Body: ${JSON.stringify(body)}`);
+    this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
     if (process.env.ENV === 'prd') {
@@ -434,99 +434,88 @@ class CatalogService implements OnModuleInit {
 
   async createTableMetadata(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+
     const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
 
-    this.logger.info(`Creating table metadata`);
-    this.logger.info(`Using endpoint: ${endpoint}`);
-    this.logger.debug(`Payload: ${JSON.stringify(body)}`);
+    this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
 
     try {
-      const { data, status } = await axios.post(endpoint, body.table_metadata);
+      const { data, status } = await axios.post(endpoint, {...body.table_metadata});
 
       this.logger.info(
-        `Table metadata created successfully with status ${status}`,
+        `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
+        {...body.logMetadata},
       );
       return data.id;
     } catch (error) {
       this.logger.error(
-        `Failed to create table metadata failed with status ${
+        `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
           error.response?.status
-        } because of ${error.response?.data || error.message}`,
-        {
-          error: error.response?.data || error.message,
-          status: error.response?.status,
-          endpoint,
-          payload: body.table_metadata,
-        },
-      );
+        } because of ${error.response?.data || error.message}`, {...body.logMetadata});
       throw new Error(error.response?.data?.message || error.message);
     }
   }
 
   async createColumnMetadata(body: any): Promise<number[]> {
     const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
     const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
 
-    this.logger.info(`Creating column metadata`);
-    this.logger.info(`Using endpoint: ${endpoint}`);
-    this.logger.debug(`Payload: ${JSON.stringify(body)}`);
+ 
 
     try {
+      this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
+      this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+      this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
       const { data, status } = await axios.post(endpoint, body.column_metadata);
 
       this.logger.info(
-        `Column metadata created successfully with status ${status}`,
+        `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
+        {...body.logMetadata},
       );
       return data.map((column) => column.id);
     } catch (error) {
       this.logger.error(
-        `Failed to create column metadata failed with status ${
+        `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
           error.response?.status
-        } because of ${error.response?.data || error.message}`,
-        {
-          error: error.response?.data || error.message,
-          status: error.response?.status,
-          endpoint,
-          payload: body.column_metadata,
-        },
-      );
+        } because of ${error.response?.data || error.message}`, {...body.logMetadata});
       throw new Error(error.response?.data?.message || error.message);
     }
   }
 
   async createDataPreview(body: any): Promise<number> {
     const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
     const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
 
-    this.logger.info(`Creating data preview`);
-    this.logger.info(`Using endpoint: ${endpoint}`);
-    this.logger.debug(`Payload: ${JSON.stringify(body)}`);
+    this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
 
     try {
       const { data, status } = await axios.post(endpoint, body.data_preview);
 
       this.logger.info(
-        `Data preview created successfully with status ${status}`,
+        `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
+        {...body.logMetadata},
       );
       return data.id;
     } catch (error) {
       this.logger.error(
-        `Failed to create data preview failed with status ${
+        `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
           error.response?.status
         } because of ${error.response?.data || error.message}`,
-        {
-          error: error.response?.data || error.message,
-          status: error.response?.status,
-          endpoint,
-          payload: body.data_preview,
-      });
+        {...body.logMetadata},
+      );
       throw new Error(error.response?.data?.message || error.message);
     }
   }
 
   async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
     const customer_name_raw = metadata.get('customer_name');
-    this.logger.debug(`Raw customer_name value: ${JSON.stringify(customer_name_raw)}`);
 
     const customer_name = customer_name_raw?.[0]?.toString();
     if (!customer_name) {