UPDATE: migrating maestro to backend nodepool

- Enhanced logs to include more detailed request body information.
- Standardized log calls to include extra metadata, such as service name, environment, customer name, user ID, HTTP method, and request path.
- Improved error logging to provide better debugging insights.

Example log format:
EXTRA[{"service":{"name":"maestro","environment":"stg"},"customer_name":"dadosfera","user_id":"047824d8-0021-709a-bf15-39b8dd069aaf","method":"POST","path":"/catalog/register-dataset"}]

Related tickets or issues: [TECN-9441]

.github/workflows/deploy-k8s.yml

@@ -1,73 +0,0 @@
-name: Deploy K8S Modifications
-
-on:
-  push:
-    branches:
-      - main
-      - beta
-
-jobs:
-  extract_environment:
-    runs-on: ubuntu-22.04
-    outputs:
-      environment: ${{ steps.extract_environment.outputs.environment }}
-    steps:
-      - name: Extract Environment
-        run: |
-          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
-            echo "environment=prd" >> $GITHUB_OUTPUT
-          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
-            echo "environment=stg" >> $GITHUB_OUTPUT
-          fi
-        id: extract_environment
-
-  helmfile-deploy:
-    needs: [extract_environment]
-    runs-on: [self-hosted, "prd-azure"]
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      - name: Set up Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: 'v3.9.0'
-
-      - name: Install Azure ClI
-        run: |
-          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
-
-      - uses: azure/login@v2
-        with:
-          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-
-      - name: Install Helmfile
-        run: |
-          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
-          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
-          mv helmfile /usr/local/bin/
-          helmfile --version
-
-      - name: Install Helm Diff Plugin
-        run: helm plugin install https://github.com/databus23/helm-diff || true
-
-      - name: Setup kubectl
-        uses: azure/setup-kubectl@v1
-        with:
-          version: 'v1.30.1'
-
-      - name: Authenticate with cluster
-        env:
-          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
-        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
-
-      - name: Run Helmfile Apply
-        env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
-        run: helmfile -f helmfiles/${ENV}.yaml sync

README.md

@@ -2,6 +2,7 @@
   <image src="./assets/maestro.svg" style="width:10rem">
 </p>
 
+
 # Maestro
 
 Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.

docsfera.json

@@ -4787,6 +4787,41 @@
           ]
         }
       },
+      "/catalog/register-dataset": {
+        "post": {
+          "operationId": "CatalogController_registerDatasetWithMetadataRequest",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "201": {
+              "description": ""
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/terms-of-use/token": {
         "get": {
           "operationId": "TermsOfUseController_getToken",

helmfiles/prd.yaml

@@ -17,4 +17,6 @@ charts:
       - name: maestro.open_customer_id
         value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
       - name: maestro.open_group_id
-        value: 401573bb-334f-44b2-b30e-88d4cea31ae9
+        value: 401573bb-334f-44b2-b30e-88d4cea31ae9
+      - name: replicaCount
+        value: 2

helmfiles/stg.yaml

@@ -18,4 +18,5 @@ charts:
         value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
       - name: maestro.open_group_id
         value: e3f98a2f-7748-4981-8505-7695c8ca8218
-        
+      - name: replicaCount
+        value: 1

maestro/templates/deployment.yaml

@@ -35,7 +35,13 @@ spec:
                   - key: application
                     operator: In
                     values:
-                      - backend
+                      - general
+
+      tolerations:
+      - key: "kubernetes.azure.com/scalesetpriority"
+        operator: "Equal"
+        value: "spot"
+        effect: "NoSchedule"
 
       containers:
         - name: maestro

maestro/values.yaml

@@ -3,7 +3,7 @@
 # Declare variables to be passed into your templates.
 
 replicaCount: 3
-hostname: maestro-temp.dadosfera.ai
+hostname: maestro.dadosfera.ai
 image:
   repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
   pullPolicy: IfNotPresent

package.json

@@ -30,7 +30,7 @@
     "@aws-sdk/client-secrets-manager": "^3.414.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.37.0-beta.5",
+    "@dadosfera/protospack-v2": "3.37.0-beta.7",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/catalog/catalog-client.ts

@@ -5,6 +5,7 @@ import {
 } from '@nestjs/microservices';
 import { credentials } from '@grpc/grpc-js';
 import { Catalog } from '@dadosfera/protospack-v2';
+import { PlatformInterfaces } from '@dadosfera/protospack-v2';
 
 const isLocalConnection =
   process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
@@ -19,11 +20,13 @@ export class CatalogClientConfiguration {
       package: [
         Catalog.ProtoPackages.ReadPackage,
         Catalog.ProtoPackages.WritePackage,
+        PlatformInterfaces.ProtoPackages.WritePackage
       ],
       credentials: isLocalConnection ? undefined : credentials.createSsl(),
       protoPath: [
         Catalog.ProtoPaths.ReadFilePath,
         Catalog.ProtoPaths.WriteFilePath,
+        PlatformInterfaces.ProtoPaths.WriteFilePath
       ],
       loader: {
         keepCase: true,

src/modules/catalog/catalog.controller.ts

@@ -13,6 +13,8 @@ import {
   Put,
   Query,
   UseFilters,
+  HttpException,
+  HttpStatus,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -30,6 +32,7 @@ import { CatalogService } from './catalog.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 import { RequestUser, User } from 'src/decorators/user.decorator';
+import { lastValueFrom } from 'rxjs';
 import {
   BatchRemoveRlsRulesRequest,
   GetDatasetCatalogTaskRes,
@@ -54,6 +57,7 @@ import {
   AddRlsRuleRequest,
   GetNimbusDashboardsRequest,
   GetRlsRulesRequest,
+  RegisterDatasetWithMetatadaRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 @ApiTags('Catalog')
@@ -743,4 +747,94 @@ export class CatalogController {
 
     return JSON.parse(dashboards);
   }
+
+
+  @Post('register-dataset')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
+  )
+  async registerDatasetWithMetadataRequest(
+    @Body() body: RegisterDatasetWithMetatadaRequest,
+    @User() user: RequestUser,
+  ) {
+      const { customer_id, customer_name, user_id, username } = user;
+      const logMetadata = {
+        customer_name: customer_name,
+        user_id: user_id,
+        method: 'POST',
+        path: '/catalog/register-dataset',
+      };
+    try {
+
+      const metadata = PackTheMetadata({
+        customer_id,
+        customer_name,
+        user_id,
+        username,
+      });
+
+      this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
+
+      // Create table metadata
+      const tableMetadataBody = {
+        table_metadata: body.table_metadata,
+        info: {
+          customer: customer_name,
+          
+        },
+        logMetadata: logMetadata
+      };
+      
+      const table_metadata_id = await this.catalogService.createTableMetadata(
+        tableMetadataBody,
+      );
+      this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
+
+      // Create column metadata
+      const columnMetadataBody = {
+        column_metadata: body.column_metadata,
+        info: {
+          customer: customer_name,
+        },
+      };
+      this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
+      const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
+
+      // Create data preview
+      const dataPreviewBody = {
+        data_preview: body.data_preview,
+        info: {
+          customer: customer_name,
+        },
+      };
+      this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
+      const data_preview_id = await this.catalogService.createDataPreview(
+        dataPreviewBody,
+      );
+
+      // Catalog dataset item
+      this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
+      await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
+      this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
+      return {
+        message: 'Dataset registered successfully',
+        table_metadata_id: table_metadata_id,
+        column_metadata_ids: column_metadata_ids,
+        data_preview_id: data_preview_id,
+      };
+    } catch (error) {
+      this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
+
+      throw new HttpException(
+        {
+          message: 'Ocorreu um erro ao registrar o dataset',
+          error: error.message,
+          code: 'REGISTRATION_FAILED',
+          details: error.message,
+        },
+        HttpStatus.INTERNAL_SERVER_ERROR,
+      );
+    }
+  }
 }

src/modules/catalog/catalog.service.ts

@@ -5,6 +5,11 @@ import {
   ProtoServices,
   Messages,
 } from '@dadosfera/protospack-v2/dist/lib/Catalog';
+import {
+  Messages as PlatformInterfaceMessages,
+  WriteService as PlatformInterfaceWriteService,
+  ProtoServices as PlatformInterfacesProtoServices,
+} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
 import {
   BadRequestException,
   HttpException,
@@ -33,6 +38,7 @@ import {
 class CatalogService implements OnModuleInit {
   catalogReadService: ReadService.CatalogReadServices;
   catalogWriteService: WriteService.CatalogWriteServices;
+  platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
   logger: any;
   constructor(
     @Inject(DadosferaLogger)
@@ -54,9 +60,14 @@ class CatalogService implements OnModuleInit {
       this.grpcClient.getService<WriteService.CatalogWriteServices>(
         ProtoServices.CatalogWriteServices,
       );
+    this.platformWriteService =
+      this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
+        PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
+      );
   }
 
   _getNimbusUrl(body) {
+    this.logger.debug(`Body: ${JSON.stringify(body)}`);
     const customer = body.info.customer.toLowerCase();
 
     if (process.env.ENV === 'prd') {
@@ -420,6 +431,112 @@ class CatalogService implements OnModuleInit {
     );
     return res.dashboards;
   }
+
+  async createTableMetadata(body: any): Promise<number> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+
+    const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
+
+    this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});  
+
+    try {
+      const { data, status } = await axios.post(endpoint, {...body.table_metadata});
+
+      this.logger.info(
+        `Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.id;
+    } catch (error) {
+      this.logger.error(
+        `Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
+          error.response?.status
+        } because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async createColumnMetadata(body: any): Promise<number[]> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
+    const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
+
+ 
+
+    try {
+      this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
+      this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+      this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
+      const { data, status } = await axios.post(endpoint, body.column_metadata);
+
+      this.logger.info(
+        `Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.map((column) => column.id);
+    } catch (error) {
+      this.logger.error(
+        `Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
+          error.response?.status
+        } because of ${error.response?.data || error.message}`, {...body.logMetadata});
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async createDataPreview(body: any): Promise<number> {
+    const nimbusUrl = this._getNimbusUrl(body);
+    this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
+    const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
+
+    this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
+    this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
+    this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
+
+    try {
+      const { data, status } = await axios.post(endpoint, body.data_preview);
+
+      this.logger.info(
+        `Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
+        {...body.logMetadata},
+      );
+      return data.id;
+    } catch (error) {
+      this.logger.error(
+        `Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
+          error.response?.status
+        } because of ${error.response?.data || error.message}`,
+        {...body.logMetadata},
+      );
+      throw new Error(error.response?.data?.message || error.message);
+    }
+  }
+
+  async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
+    const customer_name_raw = metadata.get('customer_name');
+
+    const customer_name = customer_name_raw?.[0]?.toString();
+    if (!customer_name) {
+      throw new BadRequestException('Customer name not found in metadata');
+    }
+    const res = await lastValueFrom(
+      this.platformWriteService.CatalogDataAssets(
+        {
+          data_assets: [
+            {
+              data_asset_id: table_metadata_id.toString(),
+              customer_name: customer_name,
+              data_asset_type: 'dataset',
+            },
+          ],
+        },
+        metadata,
+      ),
+    );
+    return res;
+  }
 }
 
 export { CatalogService };

src/modules/open-data/open-data.controller.ts

@@ -41,9 +41,6 @@ export class OpenDataController {
     @Req()
     request: Request,
   ) {
-    if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) {
-      throw new ForbiddenException();
-    }
     this.logger.info('createUser for open data');
 
     this.logger.info('user-agent', request.headers['user-agent']);

src/modules/pipelinesV2/pipelines.service.ts

@@ -328,7 +328,7 @@ export class PipelinesService implements OnModuleInit {
     const res = await lastValueFrom(
       this.pipelineReadService.PipelineV2GetDashboardUrl(
         {
-          dashboard_id: '83',
+          dashboard_id: '95',
           exp: '15m',
           metabase_customer_name: 'dadosferatech',
         },

src/modules/theme/theme.controller.ts

@@ -95,10 +95,10 @@ export class ThemeController {
   }
 
   private validMimeType(file: Express.Multer.File) {
-    const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png', 'image/svg+xml'];
+    const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png'];
 
     if (file && !mimeTypesValid.includes(file.mimetype)) {
-      throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, png ou svg`, HttpStatus.BAD_REQUEST);
+      throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, ou png`, HttpStatus.BAD_REQUEST);
     }
   }