FEAT: add endpoint to retrieve jobs for a specific pipeline run

.github/workflows/validate-k8s.yml

@@ -71,11 +71,6 @@ jobs:
           sudo mv helmfile /usr/local/bin/
           helmfile --version
 
-      - name: Install Helm Diff plugin
-        run: |
-          helm plugin install https://github.com/databus23/helm-diff --version v3.9.3
-          helm diff version
-
       - name: Debug Helm env
         run: |
           helm env
@@ -107,5 +102,4 @@ jobs:
       - name: Run Helmfile Diff
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
-          HELM_PLUGINS: /home/runner/.local/share/helm/plugins
         run: helmfile -f deploy/helmfiles/${ENV}.yaml diff

deploy/helm-chart/templates/deployment.yaml

@@ -113,8 +113,6 @@ spec:
               value: {{ .Values.maestro.platform_api_url }}
             - name: STORAGE_EXPLORER_API_URL
               value: {{ .Values.maestro.storage_explorer_api_url | quote }}
-            - name: FIREBASE_BASE_URL
-              value: {{ .Values.maestro.firebase_base_url }}
             - name: JWT_PRIVATE_KEY
               valueFrom:
                 secretKeyRef:

deploy/helm-chart/values-stg.yaml

@@ -10,7 +10,6 @@ maestro:
   redis_database: "1"
   platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
   storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
-  firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
 
 hostname: maestro.stg.dadosfera.ai
 

deploy/helm-chart/values.yaml

@@ -55,7 +55,6 @@ maestro:
   redis_database: "0"
   redis_tls: "true"
   cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
-  firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/prd
 autoscaling:
   enabled: false
   minReplicas: 1

docsfera.json

@@ -3343,7 +3343,14 @@
           ],
           "responses": {
             "200": {
-              "description": ""
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
             }
           },
           "tags": [
@@ -3573,10 +3580,7 @@
               "content": {
                 "application/json": {
                   "schema": {
-                    "type": "array",
-                    "items": {
-                      "type": "object"
-                    }
+                    "type": "string"
                   }
                 }
               }
@@ -3640,46 +3644,6 @@
           ]
         }
       },
-      "/pipelinesV2/{id}/upgrade": {
-        "patch": {
-          "operationId": "PipelinesController_upgradeConnector",
-          "parameters": [
-            {
-              "name": "dadosfera-lang",
-              "in": "header",
-              "required": false,
-              "schema": {
-                "enum": [
-                  "pt-br",
-                  "en-us"
-                ],
-                "type": "string"
-              }
-            },
-            {
-              "name": "id",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "responses": {
-            "204": {
-              "description": ""
-            }
-          },
-          "tags": [
-            "PipelinesV2"
-          ],
-          "security": [
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
       "/pipelinesV2/init-upload": {
         "post": {
           "operationId": "PipelinesController_initUploadFile",
@@ -3867,6 +3831,88 @@
           ]
         }
       },
+      "/pipelines/start/{id}": {
+        "post": {
+          "operationId": "PipelinesController_activate",
+          "summary": "",
+          "deprecated": true,
+          "description": "This method is deprecated. Please use route /pipelinesV2/start/:id instead",
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "201": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Pipelines"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
+      "/pipelines/{id}/status": {
+        "get": {
+          "operationId": "PipelinesController_getPipelineStatus",
+          "summary": "",
+          "deprecated": true,
+          "description": "This method is deprecated. Please use route /pipelinesV2/:id/status instead",
+          "parameters": [
+            {
+              "name": "id",
+              "required": true,
+              "in": "path",
+              "schema": {
+                "type": "string"
+              }
+            }
+          ],
+          "responses": {
+            "200": {
+              "description": "",
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "type": "object"
+                  }
+                }
+              }
+            }
+          },
+          "tags": [
+            "Pipelines"
+          ],
+          "security": [
+            {
+              "access-token": []
+            },
+            {
+              "access-token": []
+            }
+          ]
+        }
+      },
       "/transformations": {
         "post": {
           "operationId": "TransformationsController_create",
@@ -4570,50 +4616,6 @@
           ]
         }
       },
-      "/platform/pipelines/{pipelineId}/pipeline_run/{runId}/jobs": {
-        "get": {
-          "operationId": "PlatformApiController_getPipelineRunJobs",
-          "summary": "Get pipeline run jobs",
-          "parameters": [
-            {
-              "name": "pipelineId",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            },
-            {
-              "name": "runId",
-              "required": true,
-              "in": "path",
-              "schema": {
-                "type": "string"
-              }
-            }
-          ],
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          },
-          "tags": [
-            "Platform API"
-          ],
-          "security": [
-            {
-              "access-token": []
-            }
-          ]
-        }
-      },
       "/platform/jobs/{jobId}/input": {
         "put": {
           "operationId": "PlatformApiController_updateJobInput",
@@ -8591,29 +8593,6 @@
             "Health"
           ]
         }
-      },
-      "/release_note": {
-        "get": {
-          "operationId": "ReleaseNoteController_getLatestReleaseNote",
-          "parameters": [],
-          "responses": {
-            "200": {
-              "description": "",
-              "content": {
-                "application/json": {
-                  "schema": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          },
-          "security": [
-            {
-              "access-token": []
-            }
-          ]
-        }
       }
     },
     "info": {

package-lock.json

@@ -16,7 +16,8 @@
         "@aws-sdk/lib-dynamodb": "^3.414.0",
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.40.0-beta.10",
+        "@dadosfera/protospack": "2.5.3",
+        "@dadosfera/protospack-v2": "3.40.0-beta.8",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -1734,10 +1735,19 @@
         "winston-log2gelf": "^2.4.0"
       }
     },
+    "node_modules/@dadosfera/protospack": {
+      "version": "2.5.3",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack/-/protospack-2.5.3.tgz",
+      "integrity": "sha512-yOLnd+s6n9VkPpZXO8HnUY27CQPHj/qs+ecddviA4Ldn0Gx4KGRgbVdsSSP45nPm0GHhCd2bHg4ap+la7xtRmA==",
+      "license": "ISC",
+      "dependencies": {
+        "rxjs": "^7.5.5"
+      }
+    },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.40.0-beta.10",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.10.tgz",
-      "integrity": "sha512-F45dSEIKG+gwwDMYHayA242bFwhFTJbZm26KesQbGhf4I45ur6hYZD8dK0voNuVQInLMQhtm6+7th6/jJ8xpTQ==",
+      "version": "3.40.0-beta.8",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz",
+      "integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==",
       "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",

package.json

@@ -34,7 +34,8 @@
     "@aws-sdk/lib-dynamodb": "^3.414.0",
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.40.0-beta.10",
+    "@dadosfera/protospack": "2.5.3",
+    "@dadosfera/protospack-v2": "3.40.0-beta.8",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",

src/app.module.ts

@@ -17,6 +17,7 @@ import { ConnectionTestModule } from './modules/connection-test/connection-test.
 import { NetworkConfigModule } from './modules/network-config/network-config.module';
 import { InputsModule } from './modules/inputs/inputs.module';
 import { OauthModule } from './modules/oauth/oauth.module';
+import { PipelinesModule } from './modules/pipelines/pipelines.module';
 import { TransformationsModule } from './modules/transformations/transformations.module';
 import { HealthModule } from './modules/health/health.module';
 import { CatalogModule } from './modules/catalog/catalog.module';
@@ -34,8 +35,6 @@ import { ShareMetadataModule } from './modules/share-metadata/share-metadata.mod
 import { ApiKeyModule } from './modules/api-key/api-key.module';
 import { PlatformApiModule } from './modules/platform-api/platform-api.module';
 import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
-import { ReleaseNoteModule } from './modules/release_note/release_note.module';
-
 
 @Module({
   providers: [
@@ -59,6 +58,7 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
     PermissionsModule,
     TermsOfUseModule,
     ConnectionTestModule,
+    PipelinesModule,
     TransformationsModule,
     UsersModule,
     RolesModule,
@@ -79,7 +79,6 @@ import { ReleaseNoteModule } from './modules/release_note/release_note.module';
     StorageExplorerModule,
     //Always leave HealthModule last, so it is on the bottom of swagger
     HealthModule,
-    ReleaseNoteModule,
   ],
 })
 export class AppModule {}

src/modules/catalog/catalog.module.ts

@@ -5,17 +5,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { CatalogController } from './catalog.controller';
 import { CatalogClientConfiguration } from './catalog-client';
 import { ClientsModule } from '@nestjs/microservices';
+import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { UsersModule } from '../users/users.module';
 import { RolesModule } from '../roles/roles.module';
 import { CustomersModule } from '../customers/customers.module';
 import { ShareModule } from './share/share.module';
 import { CatalogService } from './catalog.service';
+import { MixpanelModule } from '../mixpanel/mixpanel.module';
 
 const client = new CatalogClientConfiguration();
 
 @Module({
   imports: [
     ClientsModule.register([client.providerOptions]),
+    OldPipelineModule,
     UsersModule,
     RolesModule,
     CustomersModule,

src/modules/inputs/dtos/old_interfaces.ts

@@ -1,8 +1,4 @@
-export interface Info {
-  user_id: string;
-  customer_id: string;
-  customer: string;
-}
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
 interface Values {
   jdbc_user: string;

src/modules/pipelines/client.service.ts

@@ -0,0 +1,78 @@
+import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
+import { ClientGrpc } from '@nestjs/microservices';
+import {
+  PipelineServicesNames,
+  PipelinesServiceInterface,
+} from '@dadosfera/protospack';
+import { lastValueFrom } from 'rxjs';
+
+import { IIdRequest } from './interfaces';
+
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { PipelinesClientConfiguration } from './pipelines-client';
+
+export class PipelinesClientService implements OnModuleInit {
+  private pipelineService: PipelinesServiceInterface;
+  logger: DadosferaLogger;
+
+  constructor(
+    @Inject(DadosferaLogger)
+      dadosferaLogger: DadosferaLogger,
+    @Inject(PipelinesClientConfiguration.name)
+    private readonly grpcClient: ClientGrpc,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  onModuleInit() {
+    this.pipelineService =
+      this.grpcClient.getService<PipelinesServiceInterface>(
+        PipelineServicesNames.PipelineService,
+      );
+  }
+
+  async getPipelineStatus(data) {
+    this.logger.info('PipelinesClientService - GetPipelineStatus');
+
+    const statusPipelineResponse = await lastValueFrom(
+      this.pipelineService.getPipelineStatus(data),
+    )
+      .then((res) => {
+        const statusArray =
+          res.status?.sort((a, b) => {
+            if (a.id < b.id) {
+              return 1;
+            } else {
+              return -1;
+            }
+          }) || [];
+        return { status: statusArray };
+      })
+      .catch((err) => {
+        this.logger.error(err.message);
+        throw new Error(err);
+      });
+    this.logger.info('Done');
+
+    return statusPipelineResponse;
+  }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    this.logger.info('PipelinesClientService - RunPipeline');
+    const statusPipelineResponse = await lastValueFrom(
+      this.pipelineService.triggerPipeline({ id, info }),
+    ).catch((err) => {
+      this.logger.error(err.message);
+      throw new Error(err);
+    });
+
+    if (statusPipelineResponse.status == false) {
+      throw new ConflictException(
+        'This pipeline is not ready yet to execute, Try again later!',
+      );
+    }
+
+    this.logger.info('Done');
+    return statusPipelineResponse;
+  }
+}

src/modules/pipelines/interfaces.d.ts

@@ -0,0 +1,36 @@
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
+
+export interface ICreatePipelineDto {
+  input: IdRequest;
+  transformations: IdRequest[];
+  output: IdRequest;
+  tags: string[];
+  name: string;
+  description: string;
+  info: Info;
+}
+
+export interface IdRequest {
+  id: string;
+}
+
+export interface IIdRequest {
+  id: string;
+  info: Info;
+}
+
+export interface IUpdatePipelineRequest {
+  input: IdRequest;
+  transformations: IdRequest[];
+  output: IdRequest;
+  tags: string[];
+  name: string;
+  description: string;
+  id: string;
+  info: Info;
+}
+
+export interface IGetPipelineLogsRequest {
+  id: string;
+  details: string;
+}

src/modules/pipelines/pipelines-client.ts

@@ -0,0 +1,33 @@
+import {
+  ClientsProviderAsyncOptions,
+  GrpcOptions,
+  Transport,
+} from '@nestjs/microservices';
+import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
+import { credentials } from '@grpc/grpc-js';
+
+const isLocalConnection =
+  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
+  process.env.PIFACTORY_URL.includes('0.0.0.0');
+
+export class PipelinesClientConfiguration {
+  public name = 'PipelinesClientConfiguration';
+  private config: GrpcOptions = {
+    transport: Transport.GRPC,
+    options: {
+      url: process.env.PIFACTORY_URL,
+      package: PipelinePackages,
+      credentials: isLocalConnection ? undefined : credentials.createSsl(),
+      protoPath: PipelineProtoFilePath,
+      loader: {
+        keepCase: true,
+        enums: String,
+        defaults: false,
+      },
+    },
+  };
+  providerOptions: ClientsProviderAsyncOptions = {
+    name: this.name,
+    ...this.config,
+  };
+}

src/modules/pipelines/pipelines.controller.ts

@@ -0,0 +1,72 @@
+import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
+import { ApiOperation, ApiTags } from '@nestjs/swagger';
+import {
+  AuthenticateCondition,
+  Authenticated,
+  RequireSomePermission,
+} from 'src/decorators/authentication.decorator';
+import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
+import { PipelinesService } from './pipelines.service';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
+
+@ApiInternalOnlyController()
+@ApiTags('Pipelines')
+@Controller('pipelines')
+@Authenticated()
+export class PipelinesController {
+  logger: DadosferaLogger;
+  constructor(
+    @Inject(DadosferaLogger)
+    dadosferaLogger: DadosferaLogger,
+    private pipelineService: PipelinesService,
+  ) {
+    this.logger = dadosferaLogger.logger;
+  }
+
+  @Post('start/:id')
+  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
+  @ApiOperation({
+    deprecated: true,
+    description:
+      'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
+  })
+  async activate(@Param('id') id: string, @Body() body) {
+    const { info } = body;
+
+    this.logger.info(
+      process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
+      {
+        user: body.info.user_id,
+        customer: body.info.customer,
+      },
+    );
+
+    const response = await this.pipelineService.runPipeline({ id, info });
+
+    return response;
+  }
+
+  @Get(':id/status')
+  @ApiOperation({
+    deprecated: true,
+    description:
+      'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
+  })
+  @RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getPipelineStatus(@Body() body, @Param('id') id: string) {
+    body.id = id;
+
+    this.logger.info(
+      process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
+      {
+        user: body.info.user_id,
+        customer: body.info.customer,
+      },
+    );
+
+    const response = await this.pipelineService.getPipelineStatus(body);
+
+    return response;
+  }
+}

src/modules/pipelines/pipelines.module.ts

@@ -0,0 +1,19 @@
+import { Module } from '@nestjs/common';
+import { ClientsModule } from '@nestjs/microservices';
+import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+
+import { PipelinesController } from './pipelines.controller';
+import { PipelinesService } from './pipelines.service';
+
+import { PipelinesClientConfiguration } from './pipelines-client';
+import { PipelinesClientService } from './client.service';
+
+const client = new PipelinesClientConfiguration();
+
+@Module({
+  imports: [ClientsModule.register([client.providerOptions])],
+  controllers: [PipelinesController],
+  providers: [PipelinesService, PipelinesClientService, DadosferaLogger],
+  exports: [PipelinesService],
+})
+export class PipelinesModule {}

src/modules/pipelines/pipelines.service.ts

@@ -0,0 +1,33 @@
+import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import { PipelinesClientService } from './client.service';
+import { IIdRequest } from './interfaces';
+import { objectCamelToSnake } from 'src/utils/CaseConverter';
+
+@Injectable()
+export class PipelinesService {
+  constructor(private pipelineClient: PipelinesClientService) {}
+
+  async getPipelineStatus(data: IIdRequest) {
+    try {
+      const pipelineStatusResponse =
+        await this.pipelineClient.getPipelineStatus(data);
+
+      return objectCamelToSnake(pipelineStatusResponse);
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    try {
+      const triggerPipelineResponse = await this.pipelineClient.runPipeline({
+        id,
+        info,
+      });
+
+      return objectCamelToSnake(triggerPipelineResponse);
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
+}

src/modules/pipelinesV2/interfaces.ts

@@ -1,4 +1,5 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
 export class PipelineInputsDTO {
   @ApiProperty()
@@ -61,12 +62,6 @@ export interface IIdRequest {
   info: Info;
 }
 
-export interface Info {
-  user_id: string;
-  customer_id: string;
-  customer: string;
-}
-
 export interface IUpdatePipelineRequest {
   input: IdRequest;
   transformations: IdRequest[];

src/modules/pipelinesV2/pipelines.controller.ts

@@ -34,6 +34,7 @@ import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
 import { RequestUser, User } from 'src/decorators/user.decorator';
 import { PackTheMetadata } from 'src/utils/PackTheMetadata';
 
+import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
 import {
   ICompleteUploadCSVFile,
   ICreatePipelineCSVFile,
@@ -62,7 +63,9 @@ export class PipelinesController {
   constructor(
     @Inject(DadosferaLogger)
     dadosferaLogger: DadosferaLogger,
+
     private pipelinesClientService: PipelinesService,
+    private oldPipelinesService: OldPipelineService,
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -198,7 +201,7 @@ export class PipelinesController {
       customer: body.info.customer,
     });
 
-    const response = await this.pipelinesClientService.getPipelineStatus(body);
+    const response = await this.oldPipelinesService.getPipelineStatus(body);
 
     return response;
   }
@@ -365,21 +368,6 @@ export class PipelinesController {
     return response;
   }
 
-  @Patch('/:id/upgrade')
-  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
-  @HttpCode(HttpStatus.NO_CONTENT)
-  async upgradeConnector(
-    @Language() language: LanguageEnum,
-    @Param('id') id: string,
-    @User() user: RequestUser
-  ) {
-    this.logger.info('PipelinesController - upgrade connector');
-
-    const metadata = PackTheMetadata(user);
-
-    await this.pipelinesClientService.upgrade(id, metadata);
-  }
-
   @Delete(':id')
   @ApiNoContentResponse()
   @HttpCode(HttpStatus.NO_CONTENT)
@@ -509,7 +497,7 @@ export class PipelinesController {
       },
     );
 
-    const response = await this.pipelinesClientService.runPipeline({ id, info });
+    const response = await this.oldPipelinesService.runPipeline({ id, info });
 
     return response;
   }

src/modules/pipelinesV2/pipelines.module.ts

@@ -7,6 +7,7 @@ import { PipelinesService } from './pipelines.service';
 
 import { PipelinesClientConfiguration } from './pipelines-client';
 
+import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { ConnectorModule } from '../connector/connector.module';
 import { InputsModule } from '../inputs/inputs.module';
 import { TransformationsModule } from '../transformations/transformations.module';
@@ -20,6 +21,7 @@ const client = new PipelinesClientConfiguration();
 @Module({
   imports: [
     ClientsModule.register([client.providerOptions]),
+    OldPipelineModule,
     ConnectorModule,
     InputsModule,
     TransformationsModule,

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,7 +1,6 @@
 /* eslint-disable no-async-promise-executor */
 import {
   BadRequestException,
-  ConflictException,
   HttpException,
   HttpStatus,
   Inject,
@@ -18,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
 
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PipelinesClientConfiguration } from './pipelines-client';
-import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
+import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
 import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
@@ -177,15 +176,6 @@ export class PipelinesService implements OnModuleInit {
     return updatePipelineResponse;
   }
 
-  async upgrade(id: string, metadata: Metadata) {
-    await lastValueFrom(
-      this.pipelineWriteService.Upgrade(
-        { id },
-        metadata,
-      ),
-    );
-  }
-
   async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
     const { id, metadata, user } = data;
     const info = {
@@ -633,49 +623,4 @@ export class PipelinesService implements OnModuleInit {
     return assets;
   }
 
-  async getPipelineStatus(data) {
-    this.logger.info('PipelinesClientService - GetPipelineStatus');
-
-    const statusPipelineResponse = await lastValueFrom(
-      this.pipelineReadService.PipelineV2GetPipelineV2Status(data),
-    )
-      .then((res) => {
-        const statusArray =
-          res.status?.sort((a, b) => {
-            if (a.id < b.id) {
-              return 1;
-            } else {
-              return -1;
-            }
-          }) || [];
-        return { status: statusArray };
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
-    this.logger.info('Done');
-
-    return statusPipelineResponse;
-  }
-
-  async runPipeline({ id, info }: IIdRequest) {
-    this.logger.info('PipelinesClientService - RunPipeline');
-    const statusPipelineResponse = await lastValueFrom(
-      this.pipelineWriteService.PipelineV2TriggerPipelineV2({ id, info }),
-    ).catch((err) => {
-      this.logger.error(err.message);
-      throw new Error(err);
-    });
-
-    if (statusPipelineResponse.status == false) {
-      throw new ConflictException(
-        'This pipeline is not ready yet to execute, Try again later!',
-      );
-    }
-
-    this.logger.info('Done');
-    return statusPipelineResponse;
-  }
-
 }

src/modules/platform-api/platform-api.controller.ts

@@ -786,6 +786,23 @@ export class PlatformApiController {
     );
   }
 
+  @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
+  @ApiOperation({ summary: 'Get pipeline run jobs' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getPipelineRunJobs(
+    @Param('pipelineId') pipelineId: string,
+    @Param('runId') runId: string,
+    @User() user: RequestUser,
+  ) {
+    const normalizedPipelineId = this.normalizePipelineId(pipelineId);
+    const normalizedRunId = this.normalizePipelineId(runId);
+    return this.platformApiService.proxy(
+      'GET',
+      `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`,
+      user,
+    );
+  }
+
   @Get('pipelines/pipeline_run/:runId/logs')
   @ApiOperation({ summary: 'Get pipeline run logs' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
@@ -832,24 +849,6 @@ export class PlatformApiController {
     );
   }
 
-  @Get('pipelines/:pipelineId/pipeline_run/:runId/jobs')
-  @ApiOperation({ summary: 'Get pipeline run jobs' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
-  async getPipelineRunJobs(
-    @Param('pipelineId') pipelineId: string,
-    @Param('runId') runId: string,
-    @User() user: RequestUser,
-  ) {
-    const normalizedPipelineId = this.normalizePipelineId(pipelineId);
-    const normalizedRunId = this.normalizePipelineId(runId);
-
-    return this.platformApiService.proxy(
-      'GET',
-      `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}/jobs`,
-      user,
-    );
-  }
-
   // ==================== JOBS - COLUMN EDITING ROUTES ====================
 
   @Put('jobs/:jobId/input')

src/modules/release_note/dto/release_note.dto.ts

@@ -1,14 +0,0 @@
-export type ReleaseNoteDTO = {
-    id: string;
-    date: string;
-    tag: string;
-    title: string;
-    visible: boolean;
-    expiryDate: string;
-    content: string;
-    showEmojis: boolean;
-    image?: string;
-    link?: string;
-    linkText?: string;
-  };
-  

src/modules/release_note/release_note.controller.spec.ts

@@ -1,20 +0,0 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { ReleaseNoteController } from './release_note.controller';
-import { ReleaseNoteService } from './release_note.service';
-
-describe('ReleaseNoteController', () => {
-  let controller: ReleaseNoteController;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      controllers: [ReleaseNoteController],
-      providers: [ReleaseNoteService],
-    }).compile();
-
-    controller = module.get<ReleaseNoteController>(ReleaseNoteController);
-  });
-
-  it('should be defined', () => {
-    expect(controller).toBeDefined();
-  });
-});

src/modules/release_note/release_note.controller.ts

@@ -1,26 +0,0 @@
-import { Controller, Get, Inject } from '@nestjs/common';
-import { ReleaseNoteService } from './release_note.service';
-import { Authenticated } from 'src/decorators/authentication.decorator';
-import { Language } from 'src/decorators/language.decorator';
-import { LanguageEnum } from 'src/utils/languages.enum';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-
-@Controller('release_note')
-@Authenticated()
-export class ReleaseNoteController {
-  logger: DadosferaLogger;
-
-  constructor(
-    @Inject(DadosferaLogger)
-    dadosferaLogger: DadosferaLogger,
-    private readonly releaseNoteService: ReleaseNoteService,
-  ) {
-    this.logger = dadosferaLogger.logger;
-  }
-
-  @Get()
-  async getLatestReleaseNote(@Language() language: LanguageEnum) {
-    this.logger.info(`Fetching latest release note for language: ${language}`);
-    return await this.releaseNoteService.getLatestReleaseNote(language);
-  }
-}

src/modules/release_note/release_note.module.ts

@@ -1,10 +0,0 @@
-import { Module } from '@nestjs/common';
-import { ReleaseNoteService } from './release_note.service';
-import { ReleaseNoteController } from './release_note.controller';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-
-@Module({
-  controllers: [ReleaseNoteController],
-  providers: [ReleaseNoteService, DadosferaLogger]
-})
-export class ReleaseNoteModule {}

src/modules/release_note/release_note.service.spec.ts

@@ -1,18 +0,0 @@
-import { Test, TestingModule } from '@nestjs/testing';
-import { ReleaseNoteService } from './release_note.service';
-
-describe('ReleaseNoteService', () => {
-  let service: ReleaseNoteService;
-
-  beforeEach(async () => {
-    const module: TestingModule = await Test.createTestingModule({
-      providers: [ReleaseNoteService],
-    }).compile();
-
-    service = module.get<ReleaseNoteService>(ReleaseNoteService);
-  });
-
-  it('should be defined', () => {
-    expect(service).toBeDefined();
-  });
-});

src/modules/release_note/release_note.service.ts

@@ -1,46 +0,0 @@
-import { Inject, Injectable } from '@nestjs/common';
-import axios, { AxiosInstance } from 'axios';
-import { LanguageEnum } from 'src/utils/languages.enum';
-import { ReleaseNoteDTO } from './dto/release_note.dto';
-import DadosferaLogger from '@dadosfera/dadosfera-logs';
-
-@Injectable()
-export class ReleaseNoteService {
-  client: AxiosInstance;
-    logger: DadosferaLogger;
-
-  constructor(
-        @Inject(DadosferaLogger)
-        dadosferaLogger: DadosferaLogger,
-  ) {
-    this.logger = dadosferaLogger.logger;
-      this.client = axios.create({
-          baseURL: process.env.FIREBASE_BASE_URL,
-      });
-  }
-
-  async getLatestReleaseNote(lang: LanguageEnum) {
-    try {
-      const lng = lang.split('-');
-      const language = lng[0] + '-' + lng[1].toUpperCase();
-
-      const endpoint = `/release_note/${language}.json`;
-      const {
-        data,
-        status,
-        config
-      } = await this.client.get<ReleaseNoteDTO>(endpoint)
-      this.logger.info(`Fetched release note for language: ${lang} with status: ${status}`);
-      this.logger.info(`Request URL: ${config.baseURL}/${config.url}`);
-
-      return data;
-    } catch (error) {
-      this.logger.error(`Error fetching release note: ${error.message}`);
-
-      if (axios.isAxiosError(error)) {
-        this.logger.error(`Axios error details: ${error.toJSON()}`);
-      }
-    }
-
-  }
-}

src/modules/transformations/interfaces.d.ts

@@ -1,8 +1,5 @@
-export interface Info {
-  user_id: string;
-  customer_id: string;
-  customer: string;
-}
+import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
+
 export interface ICreateTransformationsRequest {
   transformations: Transformation[];
   info: Info;