Merge branch 'beta' into bugfix/catalog-module

feat(connection-test): refresh connection catalog cache

.github/workflows/validate-k8s.yml

@@ -71,6 +71,11 @@ jobs:
           sudo mv helmfile /usr/local/bin/
           helmfile --version
 
+      - name: Install Helm Diff plugin
+        run: |
+          helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
+          helm diff version
+
       - name: Debug Helm env
         run: |
           helm env
@@ -102,4 +107,5 @@ jobs:
       - name: Run Helmfile Diff
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
+          HELM_PLUGINS: /home/runner/.local/share/helm/plugins
         run: helmfile -f deploy/helmfiles/${ENV}.yaml diff

deploy/helm-chart/values.yaml

@@ -47,7 +47,6 @@ maestro:
   open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
   open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
   platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
-  connections_api_url: https://wv6fw7ayj4.execute-api.us-east-1.amazonaws.com
   storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
   dedicated_proxy: ""
   restricted_ip: ""

docsfera.json

@@ -4574,6 +4574,7 @@
         "get": {
           "operationId": "PlatformApiController_getPipelineRunJobs",
           "summary": "Get pipeline run jobs",
+          "description": "Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.",
           "parameters": [
             {
               "name": "pipelineId",
@@ -4594,11 +4595,22 @@
           ],
           "responses": {
             "200": {
-              "description": "",
+              "description": "DB-backed job runs for the selected pipeline run.",
               "content": {
                 "application/json": {
                   "schema": {
-                    "type": "object"
+                    "type": "object",
+                    "properties": {
+                      "jobs": {
+                        "type": "array",
+                        "items": {
+                          "type": "object"
+                        }
+                      }
+                    },
+                    "required": [
+                      "jobs"
+                    ]
                   }
                 }
               }
@@ -5526,6 +5538,48 @@
           ]
         }
       },
+      "/catalog/custom-properties": {
+        "get": {
+          "operationId": "CatalogController_getCustomPropertyDefinitions",
+          "parameters": [
+            {
+              "name": "dadosfera-lang",
+              "in": "header",
+              "required": false,
+              "schema": {
+                "enum": [
+                  "pt-br",
+                  "en-us"
+                ],
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Catalog"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/catalog/data-asset/{id}": {
         "get": {
           "operationId": "CatalogController_getDataAsset",
@@ -11033,6 +11087,37 @@
             "docs"
           ]
         },
+        "CustomPropertyDto": {
+          "type": "object",
+          "properties": {
+            "key": {
+              "type": "string"
+            },
+            "value": {
+              "type": "string"
+            },
+            "type": {
+              "type": "string",
+              "enum": [
+                "text",
+                "number",
+                "date",
+                "boolean"
+              ]
+            },
+            "color": {
+              "type": "string"
+            },
+            "emoji": {
+              "type": "string"
+            }
+          },
+          "required": [
+            "key",
+            "value",
+            "type"
+          ]
+        },
         "IUpdateDataRequest": {
           "type": "object",
           "properties": {
@@ -11061,6 +11146,12 @@
             },
             "docs": {
               "type": "string"
+            },
+            "custom_properties": {
+              "type": "array",
+              "items": {
+                "$ref": "#/components/schemas/CustomPropertyDto"
+              }
             }
           },
           "required": [

package-lock.json

@@ -16,7 +16,7 @@
         "@aws-sdk/lib-dynamodb": "^3.414.0",
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "^3.40.0-beta.10",
+        "@dadosfera/protospack-v2": "^3.40.0-beta.14",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1735,9 +1735,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.40.0-beta.10",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
-      "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
+      "version": "3.40.0-beta.14",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
+      "integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"

package.json

@@ -34,7 +34,7 @@
     "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "^3.40.0-beta.10",
+    "@dadosfera/protospack-v2": "^3.40.0-beta.14",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/modules/catalog/catalog.controller.ts

@@ -271,6 +271,23 @@ export class CatalogController {
 
   }
 
+  @Get('custom-properties')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
+  async getCustomPropertyDefinitions(@User() user: RequestUser) {
+    const { customer_id, customer_name, user_id, username } = user;
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+    });
+
+    return this.catalogService.getCustomPropertyDefinitions(metadata);
+  }
+
   @Get('data-asset/:id')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.CATALOG.permissions.GET,
@@ -1002,4 +1019,4 @@ export class CatalogController {
       this.logger.error(error.message);
     }
   }
-}
+}

src/modules/catalog/catalog.service.ts

@@ -120,6 +120,10 @@ class CatalogService implements OnModuleInit {
     }
   }
 
+  async getCustomPropertyDefinitions(metadata: Metadata) {
+    return lastValueFrom(this.catalogReadService.GetCustomPropertyDefinitions({}, metadata));
+  }
+
   async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
     this.logger.info('CatalogService - Manage Data assets permissions');
     if (!data.embed) data.embed = undefined;

src/modules/catalog/dtos/index.ts

@@ -1,5 +1,10 @@
 import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
-import { IsEnum } from 'class-validator';
+import {
+  IsEnum,
+  IsNotEmpty,
+  IsOptional,
+  IsString,
+} from 'class-validator';
 import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
 
 export enum DataAssetShareType {
@@ -198,6 +203,27 @@ export class IData {
   day_opening: number;
 }
 
+
+export enum CustomPropertyType {
+  TEXT = 'text',
+  NUMBER = 'number',
+  DATE = 'date',
+  BOOLEAN = 'boolean',
+}
+
+export class CustomPropertyDto {
+  @ApiProperty()
+  key: string;
+  @ApiProperty()
+  value: string;
+  @ApiProperty({ enum: CustomPropertyType })
+  type: CustomPropertyType;
+  @ApiPropertyOptional()
+  color?: string;
+  @ApiPropertyOptional()
+  emoji?: string;
+}
+
 export class IUpdateDataRequest {
   @ApiProperty()
   name: string;
@@ -211,6 +237,8 @@ export class IUpdateDataRequest {
   share_type?: DataAssetShareType;
   @ApiPropertyOptional()
   docs?: string;
+  @ApiPropertyOptional({ type: [CustomPropertyDto] })
+  custom_properties?: CustomPropertyDto[];
 }
 
 export class IUpdateCertificationStatusRequest {
@@ -372,4 +400,4 @@ export type CreateDataDocsDTO = {
   docs: string;
   asset_type: string;
 
-}
+}

src/modules/connection-test/connection-test-client.config.ts

@@ -6,9 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionTest } from '@dadosfera/protospack-v2';
 
-const infactoryUrl = process.env.INFACTORY_URL ?? '';
 const isLocalConnection =
-  infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectionTestClientConfiguration {
   private config: GrpcOptions = {

src/modules/connection/client.config.ts

@@ -6,9 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionManager } from '@dadosfera/protospack-v2';
 
-const infactoryUrl = process.env.INFACTORY_URL ?? '';
 const isLocalConnection =
-  infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectionClientConfiguration {
   public name = 'ConnectionClientConfiguration';

src/modules/connector/client.config.ts

@@ -6,9 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectorManager } from '@dadosfera/protospack-v2';
 
-const infactoryUrl = process.env.INFACTORY_URL ?? '';
 const isLocalConnection =
-  infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectorClientConfiguration {
   public name = 'ConnectorClientConfiguration';

src/modules/inputs/inputs-client.config.ts

@@ -6,9 +6,9 @@ import {
   type GrpcOptions,
 } from '@nestjs/microservices';
 
-const infactoryUrl = process.env.INFACTORY_URL ?? '';
 const isLocalConnection =
-  infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 export class InputsGrpcClient {
   public readonly name = 'InputsGrpcClient';
   private config: GrpcOptions = {

src/modules/inputs/inputs.service.ts

@@ -23,6 +23,7 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 import { CreateInputReq } from './dtos/input.model';
+import { Metadata } from '@grpc/grpc-js';
 
 @Injectable()
 
@@ -73,10 +74,10 @@ export class InputsService {
       objectCamelToSnake(createInputResponse);
       return createInputResponse;
     },
-    update: async (updateInputDTO: UpdateInputRequest): Promise<InputUpdateResponse> => {
+    update: async (updateInputDTO: UpdateInputRequest, metadata: Metadata): Promise<InputUpdateResponse> => {
       this.logger.info('InputClientService - Update' + JSON.stringify(updateInputDTO));
       const updateInputResponse = await lastValueFrom(
-        this.inputWriteService.InputUpdate(updateInputDTO),
+        this.inputWriteService.InputUpdate(updateInputDTO, metadata),
       );
 
       return updateInputResponse;
@@ -206,7 +207,7 @@ export class InputsService {
     return findOneInputResponse;
   }
 
-  async update(id: string, data, info: Info) {
+  async update(id: string, data, info: Info, metadata?: Metadata) {
     // this.validateCron({ ...data, info });
     try {
       const {
@@ -217,7 +218,7 @@ export class InputsService {
         id,
         ...data,
         info,
-      });
+      }, metadata);
 
       const updateInputResponse = this.adjustInputPayload(
         input,

src/modules/network-config/network-config-client.config.ts

@@ -6,9 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { NetworkConfig } from '@dadosfera/protospack-v2';
 
-const infactoryUrl = process.env.INFACTORY_URL ?? '';
 const isLocalConnection =
-  infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class NetworkConfigGrpcClient {
   public name = 'NetworkConfigGrpcClient';

src/modules/pipelinesV2/pipelines.controller.ts

@@ -317,7 +317,6 @@ export class PipelinesController {
   ) {
     this.logger.info('PipelinesController - update', { user });
 
-    const { customer_id, customer_name, user_id, username } = user;
     const info: Info = {
       user_id: user.user_id,
       customer: user.customer_name,
@@ -325,13 +324,7 @@ export class PipelinesController {
       pipeline_id: pipelineId
     };
 
-    const metadata = PackTheMetadata({
-      customer_id,
-      customer_name,
-      user_id,
-      username,
-      language,
-    });
+    const metadata = PackTheMetadata(user);
 
     const response = await this.pipelinesClientService.updatePipelineInput(
       pipelineId,

src/modules/pipelinesV2/pipelines.service.ts

@@ -383,7 +383,8 @@ export class PipelinesService implements OnModuleInit {
     const updateInputResponse = await this.inputsService.update(
       inputId,
       updateInputDTO,
-      info
+      info,
+      metadata
     );
 
     const inputRollback = () => {
@@ -404,34 +405,36 @@ export class PipelinesService implements OnModuleInit {
 
     const nimbusUpdates = updateInputResponse?.tablesUpdate || [];
 
-    nimbusUpdates.forEach(update => {
-      const nimbusRollback = () => {
-        return this.nimbusService.renameTable(
-          info.customer,
-          update.database,
-          {
-            table_name: update.table_name,
-            table_schema: update.table_schema
-          },
-          {
-            table_name: update.old_table_name,
-            table_schema: update.old_table_schema
-          }
-        );
+    if (user.customer_modules.includes('catalog')) {
+      nimbusUpdates.forEach(update => {
+        const nimbusRollback = () => {
+          return this.nimbusService.renameTable(
+            info.customer,
+            update.database,
+            {
+              table_name: update.table_name,
+              table_schema: update.table_schema
+            },
+            {
+              table_name: update.old_table_name,
+              table_schema: update.old_table_schema
+            }
+          );
+        }
+        rollback.push(nimbusRollback);
+      });
+  
+      try {
+        await this.updateNimbus(info.customer, nimbusUpdates);
+      } catch (error) {
+        this.logger.error(error);
+        if (error instanceof AxiosError) {
+          this.logger.error(JSON.stringify(error.response.data));
+        }
+        await this.executeRenameRollback(rollback);
+  
+        throw new Error("Error Nimbus updating tables");
       }
-      rollback.push(nimbusRollback);
-    });
-
-    try {
-      await this.updateNimbus(info.customer, nimbusUpdates);
-    } catch (error) {
-      this.logger.error(error);
-      if (error instanceof AxiosError) {
-        this.logger.error(JSON.stringify(error.response.data));
-      }
-      await this.executeRenameRollback(rollback);
-
-      throw new Error("Error Nimbus updating tables");
     }
 
     try {

src/modules/platform-api/platform-api.controller.ts

@@ -14,7 +14,7 @@ import {
   NotFoundException,
   UseGuards,
 } from '@nestjs/common';
-import { ApiTags, ApiOperation } from '@nestjs/swagger';
+import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 
 import {
@@ -72,6 +72,10 @@ export class PlatformApiController {
     return id?.replace(/-/g, '_') || '';
   }
 
+  private decodePathParam(value: string): string {
+    return value ? decodeURIComponent(value) : '';
+  }
+
   /**
    * Denormalize ID back to UUID format (replace _ with -).
    * Used when we receive a normalized ID but need the original UUID.
@@ -832,6 +836,40 @@ export class PlatformApiController {
     );
   }
 
+  @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
+  @ApiOperation({
+    summary: 'Get pipeline run jobs',
+    description: 'Proxies platform-api DB-backed job runs and returns `{ jobs: [...] }`.',
+  })
+  @ApiOkResponse({
+    description: 'DB-backed job runs for the selected pipeline run.',
+    schema: {
+      type: 'object',
+      properties: {
+        jobs: {
+          type: 'array',
+          items: { type: 'object' },
+        },
+      },
+      required: ['jobs'],
+    },
+  })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getPipelineRunJobs(
+    @Param('pipelineId') pipelineId: string,
+    @Param('runId') runId: string,
+    @User() user: RequestUser,
+  ) {
+    const normalizedPipelineId = this.normalizePipelineId(pipelineId);
+    const decodedRunId = this.decodePathParam(runId);
+
+    return this.platformApiService.proxy(
+      'GET',
+      `/pipeline/${normalizedPipelineId}/pipeline_run/${decodedRunId}/jobs`,
+      user,
+    );
+  }
+
   // ==================== JOBS - COLUMN EDITING ROUTES ====================
 
   @Put('jobs/:jobId/input')

src/modules/release_note/release_note.controller.spec.ts

@@ -1,7 +1,6 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteController } from './release_note.controller';
 import { ReleaseNoteService } from './release_note.service';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteController', () => {
   let controller: ReleaseNoteController;
@@ -9,7 +8,7 @@ describe('ReleaseNoteController', () => {
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
       controllers: [ReleaseNoteController],
-      providers: [ReleaseNoteService, DadosferaLogger],
+      providers: [ReleaseNoteService],
     }).compile();
 
     controller = module.get<ReleaseNoteController>(ReleaseNoteController);

src/modules/release_note/release_note.service.spec.ts

@@ -1,13 +1,12 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { ReleaseNoteService } from './release_note.service';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
 
 describe('ReleaseNoteService', () => {
   let service: ReleaseNoteService;
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [ReleaseNoteService, DadosferaLogger],
+      providers: [ReleaseNoteService],
     }).compile();
 
     service = module.get<ReleaseNoteService>(ReleaseNoteService);

src/modules/transformations/transformations-client.ts

@@ -6,9 +6,9 @@ import {
 import { credentials } from '@grpc/grpc-js';
 import { Transformation } from '@dadosfera/protospack-v2';
 
-const infactoryUrl = process.env.INFACTORY_URL ?? '';
 const isLocalConnection =
-  infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class TransformationsClientConfiguration {
   public name = 'TransformationsClientConfiguration';

src/utils/PackTheMetadata.ts

@@ -9,6 +9,7 @@ interface IMetadata {
   details?: string;
   sensitive?: string;
   roles?: string[];
+  customer_modules?: string[];
   is_data_manager?: boolean;
   access_token?: string;
   host?: string;