Merge pull request #443 from dadosfera/feat/qualify

Feat/qualify

deploy/helm-chart/values-stg.yaml

@@ -9,7 +9,7 @@ maestro:
   cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
   redis_database: "1"
   platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
-  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
+  storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local/api"
 
 hostname: maestro.stg.dadosfera.ai
 

package-lock.json

@@ -17,7 +17,7 @@
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "^3.39.0",
+        "@dadosfera/protospack-v2": "3.38.0-beta.28",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -47,7 +47,6 @@
         "jwk-to-pem": "^2.0.5",
         "mixpanel": "^0.17.0",
         "ms": "^3.0.0-canary.1",
-        "multer": "^2.0.2",
         "openid-client": "^5.7.1",
         "passport": "^0.6.0",
         "passport-facebook": "^3.0.0",
@@ -1745,9 +1744,10 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.39.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.39.0.tgz",
-      "integrity": "sha512-z6q+ZRpbDT6apfmly1+X818DgAFUCilFO0ATP7xR5+SH+GAUAJLY1dJRQDEkq5vFIfXnC2Nmm9aODZAs+9/5vw==",
+      "version": "3.38.0-beta.28",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
+      "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
+      "license": "ISC",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"
@@ -2928,20 +2928,6 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/@nestjs/platform-express/node_modules/concat-stream": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
-      "engines": [
-        "node >= 0.8"
-      ],
-      "dependencies": {
-        "buffer-from": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^2.2.2",
-        "typedarray": "^0.0.6"
-      }
-    },
     "node_modules/@nestjs/platform-express/node_modules/content-disposition": {
       "version": "0.5.4",
       "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -3111,24 +3097,6 @@
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
       "license": "MIT"
     },
-    "node_modules/@nestjs/platform-express/node_modules/multer": {
-      "version": "1.4.4-lts.1",
-      "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
-      "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
-      "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
-      "dependencies": {
-        "append-field": "^1.0.0",
-        "busboy": "^1.0.0",
-        "concat-stream": "^1.5.2",
-        "mkdirp": "^0.5.4",
-        "object-assign": "^4.1.1",
-        "type-is": "^1.6.4",
-        "xtend": "^4.0.0"
-      },
-      "engines": {
-        "node": ">= 6.0.0"
-      }
-    },
     "node_modules/@nestjs/platform-express/node_modules/negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3153,25 +3121,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/@nestjs/platform-express/node_modules/readable-stream": {
-      "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
-      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
-      "dependencies": {
-        "core-util-is": "~1.0.0",
-        "inherits": "~2.0.3",
-        "isarray": "~1.0.0",
-        "process-nextick-args": "~2.0.0",
-        "safe-buffer": "~5.1.1",
-        "string_decoder": "~1.1.1",
-        "util-deprecate": "~1.0.1"
-      }
-    },
-    "node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
-    },
     "node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3246,19 +3195,6 @@
         "node": ">= 0.8"
       }
     },
-    "node_modules/@nestjs/platform-express/node_modules/string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "dependencies": {
-        "safe-buffer": "~5.1.0"
-      }
-    },
-    "node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
-      "version": "5.1.2",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
-      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
-    },
     "node_modules/@nestjs/platform-express/node_modules/tslib": {
       "version": "2.5.3",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
@@ -6573,6 +6509,7 @@
       "engines": [
         "node >= 6.0"
       ],
+      "license": "MIT",
       "dependencies": {
         "buffer-from": "^1.0.0",
         "inherits": "^2.0.3",
@@ -9191,11 +9128,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
-    },
     "node_modules/isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -10793,6 +10725,7 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
       "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
+      "license": "MIT",
       "dependencies": {
         "append-field": "^1.0.0",
         "busboy": "^1.6.0",
@@ -11795,11 +11728,6 @@
         "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
-    "node_modules/process-nextick-args": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
-      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
-    },
     "node_modules/process-warning": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",

package.json

@@ -35,7 +35,7 @@
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "^3.39.0",
+    "@dadosfera/protospack-v2": "3.38.0-beta.28",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",
@@ -65,7 +65,6 @@
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
     "ms": "^3.0.0-canary.1",
-    "multer": "^2.0.2",
     "openid-client": "^5.7.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
@@ -81,6 +80,7 @@
     "swagger-ui-express": "^4.6.3"
   },
   "overrides": {
+    "multer": "2.0.2",
     "form-data": "^4.0.4",
     "body-parser": "^1.20.3",
     "cross-spawn": "^7.0.5",

src/modules/auth/auth.controller.ts

@@ -502,7 +502,7 @@ export class AuthController {
     const accessToken = req.cookies['ddf-auth'];
     const refreshToken = req.cookies['ddf-refresh-auth'];
     const userId = req.cookies['ddf-user-id'];
-    const resourceHost = req.headers["x-original-url"] as string || "" ;
+    const resourceHost = req.headers["host"]
 
     const hasUserSession = Boolean(accessToken) && Boolean(userId);
     this.logger.info('Has User Session: ' + hasUserSession);

src/modules/customers/customers.service.ts

@@ -9,12 +9,12 @@ import {
 } from '@nestjs/common';
 
 import { firstValueFrom, lastValueFrom } from 'rxjs';
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import { DucClient } from '../duc/client.config';
 import { ClientGrpc } from '@nestjs/microservices';
 import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
-import { CustomerLinksConfig } from './dtos/customers';
 import ErrorCodes from 'src/utils/errorCodes';
 import jwt from 'jsonwebtoken';
 import {
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
     )
   }
 
-  async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
+  async getLinks(customerId: string) {
     try {
       const result = await lastValueFrom(
-        this.customerService.CustomerGetLinks({ customerId }),
+        this.customerService.CustomerFindOneById({ id: customerId }),
       );
-      return (result.links as CustomerLinksConfig) || null;
+      return result.customer?.links || [];
     } catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
         throw new HttpException(err.details, HttpStatus.NOT_FOUND);
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
     }
   }
 
-  async setLinks(customerId: string, links: CustomerLinksConfig) {
+  async setLinks(customerId: string, links: Link[]) {
     if (!customerId || !links) {
       throw new HttpException(null, HttpStatus.BAD_REQUEST);
     }
 
     try {
       return await firstValueFrom(
-        this.customerService.CustomerSetLinks({
-          customerId,
-          links: links as CustomerSetLinksRequest['links'],
-        }),
+        this.customerService.CustomerUpdate({
+          id: customerId,
+          links,
+        } as CustomerUpdateRequest),
       );
     } catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)

src/modules/customers/dtos/customers.ts

@@ -1,6 +1,7 @@
+import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
-export class CustomerLinkItem {
+export class CustomerLink implements Link {
   @ApiProperty()
   href: string;
   @ApiProperty()
@@ -8,59 +9,15 @@ export class CustomerLinkItem {
   @ApiProperty()
   description: string;
   @ApiPropertyOptional()
-  iconSrc?: string;
+  iconSrc: string;
 }
-
-export class CustomerSidebarLinkItem {
-  @ApiProperty()
-  type: 'link';
-  @ApiProperty({ type: Object })
-  title: Record<string, string>;
-  @ApiProperty()
-  link: string;
-  @ApiPropertyOptional()
-  icon?: string;
-}
-
-export class CustomerSidebarMenuItem {
-  @ApiProperty()
-  type: 'menu';
-  @ApiProperty({ type: Object })
-  title: Record<string, string>;
-  @ApiPropertyOptional()
-  icon?: string;
-  @ApiProperty({ type: [CustomerSidebarLinkItem] })
-  items: CustomerSidebarLinkItem[];
-}
-
-export class CustomerSidebarSection {
-  @ApiProperty({ type: Object })
-  title: Record<string, string>;
-  @ApiProperty({
-    type: 'array',
-    items: {
-      oneOf: [
-        { $ref: '#/components/schemas/CustomerSidebarMenuItem' },
-        { $ref: '#/components/schemas/CustomerSidebarLinkItem' },
-      ],
-    },
-  })
-  items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
-}
-
-export class CustomerLinksConfig {
-  @ApiPropertyOptional({ type: [CustomerLinkItem] })
-  home?: CustomerLinkItem[];
-  @ApiPropertyOptional({ type: [CustomerSidebarSection] })
-  sidebar?: CustomerSidebarSection[];
-}
-
 export class CustomerLinkRequest {
-  @ApiProperty({ type: CustomerLinksConfig })
-  links: CustomerLinksConfig;
+  @ApiProperty({ type: [CustomerLink] })
+  links: CustomerLink[];
 }
 
 export class CustomerLinksResponse {
-  @ApiPropertyOptional({ type: CustomerLinksConfig })
-  links?: CustomerLinksConfig;
-}
+  @ApiProperty({ type: [CustomerLink] })
+  links: CustomerLink[];
+}
+

src/modules/inputs/dtos/input.model.ts

@@ -11,19 +11,17 @@ export class TableColumns {
   name: string;
   @ApiProperty()
   columns: string[];
-  @ApiPropertyOptional({ type: [Column] })
+  @ApiProperty()
   references: Column[];
   @ApiProperty()
+  identifier_columns: string[];
+  @ApiProperty()
   destination: Record<'raw' | 'qualify', {
     table_name: string;
     table_schema: string;
   }> | null;
   @ApiProperty()
   type: string;
-  @ApiPropertyOptional({ type: [String] })
-  identifier_columns?: string[];
-  @ApiPropertyOptional({ type: Column })
-  reference_column?: Column;
 }
 export class AvailableEntity {
   @ApiProperty()

src/modules/inputs/inputs.service.ts

@@ -164,11 +164,6 @@ export class InputsService {
     const inputCreateGenericRequest: InputCreateGenericRequest = {
       input: {
         ...body,
-        tables: (body.tables || []).map((table) => ({
-          ...table,
-          identifier_columns: table.identifier_columns || [],
-          reference_column: table.reference_column || table.references?.[0],
-        })),
       },
       info,
     };
@@ -205,7 +200,7 @@ export class InputsService {
   }
 
   async update(id: string, data, info: Info) {
-    this.validateCron({ ...data, info });
+    // this.validateCron({ ...data, info });
     try {
       const updateInputResponse: any = await this.OLD_inputClient.update({
         id,

src/modules/pipelinesV2/interfaces.ts

@@ -1,6 +1,15 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
 import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
+export class PipelineInputsDTO {
+  @ApiProperty()
+  tables: Array<{
+    name: string,
+    type: string,
+    
+  }>
+}
+
 export class IPipelineV2 {
   @ApiProperty()
   id: string;
@@ -123,3 +132,30 @@ export class PipelineFindAllReq {
   @ApiPropertyOptional()
   type?: string | undefined;
 }
+
+export interface UpdateTableDTO {
+  name: string;
+  type: string;
+  columns: string[];
+  destinations: {
+    raw: {
+      table_schema: string;
+      table_name: string;
+    };
+    qualify: {
+      table_schema: string;
+      table_name: string;
+    };
+  };
+  identifier_columns: string[];
+  reference_column: {
+    name: string;
+    type: string;
+  };
+  memory: number;
+}
+
+export interface UpdatePlatformInputRequest {
+  cron: string;
+  tables: Array<UpdateTableDTO>;
+}

src/modules/pipelinesV2/pipelines.controller.ts

@@ -43,11 +43,15 @@ import {
   IPipelineV2,
   IInitUploadCSVFile,
   PipelineFindAllReq,
+  UpdatePlatformInputRequest,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
+import { TableColumns } from '../inputs/dtos/input.model';
+import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 
 @ApiTags('PipelinesV2')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -223,6 +227,7 @@ export class PipelinesController {
       .then((res) => {
         //{pipeline:{tables: {tables: [], input_id: ''}}}
         let tables = JSON.parse(res.pipeline.config.tables);
+        const input_id = tables?.input_id;
         if (tables?.tables) tables = tables.tables;
         Object.assign(res.pipeline, {
           transformations: res.pipeline.transformations
@@ -231,6 +236,7 @@ export class PipelinesController {
           config: {
             cron: res.pipeline.config.cron,
             tables,
+            input_id
           },
           properties: res.pipeline.properties
             ? JSON.parse(res.pipeline.properties)
@@ -277,6 +283,45 @@ export class PipelinesController {
     return response;
   }
 
+  @Patch('/:pipelineId/inputs/:id')
+  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  async updatePipelineInput(
+    @Language() language: LanguageEnum,
+    @Body() pipelineInputDTO: UpdatePlatformInputRequest,
+    @Param('id') inputId: string,
+    @Param('pipelineId') pipelineId: string,
+    @User() user: RequestUser,
+  ) {
+    this.logger.info('PipelinesController - update', { user });
+
+
+    const { customer_id, customer_name, user_id, username } = user;
+    const info: Info = {
+      user_id: user.user_id,
+      customer: user.customer_name,
+      customer_id: user.customer_id,
+    };
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
+    });
+
+    const response = await this.pipelinesClientService.updatePipelineInput(
+      pipelineId,
+      inputId,
+      pipelineInputDTO,
+      info,
+      user,
+      metadata,
+    );
+
+    this.logger.info('PipelinesController - update: OK', { user });
+    return response;
+  }
+
   @ApiInternalOnlyEndpoint()
   @Put('/:id')
   @ApiOperation({

src/modules/pipelinesV2/pipelines.module.ts

@@ -11,6 +11,7 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe
 import { ConnectorModule } from '../connector/connector.module';
 import { InputsModule } from '../inputs/inputs.module';
 import { TransformationsModule } from '../transformations/transformations.module';
+import { PlatformApiModule } from '../platform-api/platform-api.module';
 
 const client = new PipelinesClientConfiguration();
 
@@ -21,6 +22,7 @@ const client = new PipelinesClientConfiguration();
     ConnectorModule,
     InputsModule,
     TransformationsModule,
+    PlatformApiModule
   ],
   controllers: [PipelinesController],
   providers: [PipelinesService, DadosferaLogger],

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,3 +1,4 @@
+/* eslint-disable no-async-promise-executor */
 import {
   BadRequestException,
   HttpException,
@@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
 
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PipelinesClientConfiguration } from './pipelines-client';
-import { ICreatePipelineV2Req } from './interfaces';
+import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
 import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
@@ -26,6 +27,8 @@ import { TransformationsService } from '../transformations/transformations.servi
 import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
 import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
+import { PlatformApiService } from '../platform-api/platform-api.service';
+import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -39,6 +42,7 @@ export class PipelinesService implements OnModuleInit {
     private readonly connectorService: ConnectorClientService,
     private readonly inputsService: InputsService,
     private readonly transformationsService: TransformationsService,
+    private readonly platformAPI: PlatformApiService
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -138,6 +142,7 @@ export class PipelinesService implements OnModuleInit {
     const findOnePipelineResponse = await lastValueFrom(
       this.pipelineReadService.PipelineV2FindOne(data, metadata),
     );
+    console.log('pipeline find one response', findOnePipelineResponse);
     this.logger.info('Done');
 
     return findOnePipelineResponse;
@@ -339,4 +344,137 @@ export class PipelinesService implements OnModuleInit {
 
     return res;
   }
+
+  async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
+    this.logger.info('InputClientService - Update');
+
+    this.logger.info('Update Dynamo Reference');
+    const pipelineIdFormat = pipelineId.split('-').join('_');
+    const updateInputResponse = await this.inputsService.update(
+      inputId,
+      updateInputDTO,
+      info
+    )
+
+    const requests = [];
+
+    this.logger.info('Dynamo Response', updateInputResponse);
+
+    for (const [index, table] of updateInputDTO.tables.entries()) {
+      const id = `${pipelineIdFormat}_${index}`;
+      this.logger.info('Updating input reference for table', table.name);
+      const body = {}
+      if (table.columns) {
+        body['column_include_list'] = table.columns;
+      }
+
+      if (table.reference_column) {
+        body['incremental_column_name'] = table.reference_column.name;
+        body['incremental_column_type'] = table.reference_column.type;
+      }
+
+      if (table.identifier_columns) {
+        body['primary_keys'] = table.identifier_columns;
+      }
+
+      this.logger.info('Request body', body);
+      const updateCollumns = this.platformAPI.proxy(
+        'PATCH',
+        `/jobs/${id}/input`,
+        user,
+        body
+      )
+      requests.push(updateCollumns);
+
+      if (table.memory) {
+        this.logger.info('Updating memory allocation for table', table.name);
+        const updateMemory = this.platformAPI.proxy(
+          'PUT',
+          `/jobs/${id}/memory`,
+          user,
+          {
+            amount: table.memory
+          }
+        )
+        requests.push(updateMemory);
+      }
+
+      if (table.type) {
+          const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
+          requests.push(updateSyncMode);
+      }
+    }
+
+    this.logger.info('Create Platform Request for each JOB');
+
+    if (updateInputDTO.cron) {
+      const crnUpdatedRequest = new Promise(async (resolve, reject) => {
+        const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
+
+        if (response.error) {
+          this.logger.error('Error updating pipeline cron', response.error);
+          return reject(new ErrorBuilder(response.error));
+        }
+        this.logger.error('Pipeline cron updated successfully', response);
+        return resolve(response);
+      });
+      requests.push(crnUpdatedRequest);
+    }
+
+    this.logger.info('Executing all request for the platform api');
+
+    const results = await Promise.allSettled(requests);
+    this.logger.info('Platform api response', results);
+
+    return updateInputResponse;
+    
+  }
+
+  private async updatePipelineSyncMode(table: UpdateTableDTO,  pipelineId: string, user: RequestUser) {
+    const body = {
+      target_load_type: table.type
+    }
+
+    if (table.type === 'incremental_with_qualify') {
+      body['incremental_column_name'] = table.reference_column.name;
+      body['incremental_column_type'] = table.reference_column.type;
+      body['primary_keys'] = table.identifier_columns;
+    }
+
+    if (table.type === 'incremental') {
+      body['incremental_column_name'] = table.reference_column.name;
+      body['incremental_column_type'] = table.reference_column.type;
+    }
+
+    this.logger.info('Updating pipeline sync mode', {
+      pipelineId,
+      body
+    });
+
+    return this.platformAPI.proxy(
+      "POST",
+      `/jobs/jdbc/${pipelineId}/sync-mode`,
+      user,
+      body
+    )
+  }
+
+  private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
+    try {
+      const response = await this.platformAPI.proxy(
+        'PATCH',
+        `/pipeline/${pipelineId}`,
+        user,
+        {
+          cron
+        }
+      );
+
+      return response
+    } catch (error) {
+        return {
+          error: error.message
+        }
+    }
+  }
 }

src/modules/platform-api/platform-api.controller.ts

@@ -25,6 +25,7 @@ import { ElasticsearchService } from '../../services/elasticsearch';
 import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
 import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
+import { ValidationTableDTO } from './platform-api.dto';
 
 
 type ValidateTablesDTO = {
@@ -728,26 +729,10 @@ export class PlatformApiController {
     );
   }
 
-  // ==================== Catalog ROUTES ====================
-
-  @Get('pipelines/catalog/tables')
-  @ApiOperation({ summary: 'Get all tables available' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
-  async getAvailableTables(
-    @User() user: RequestUser,
-    @Query() query: Record<string, string>,
-  ) {
-    return this.platformApiService.proxy(
-      'GET',
-      '/catalog/tables',
-      user,
-      undefined,
-      query,
-    );
-  }
+  // ==================== PIPELINE VALIDATION ====================
 
   @Get('pipelines/catalog/schemas')
-  @ApiOperation({ summary: 'Get all schemas available' })
+  @ApiOperation({ summary: 'Get available schemas' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async getAvailableSchemas(
     @User() user: RequestUser,
@@ -755,7 +740,7 @@ export class PlatformApiController {
   ) {
     return this.platformApiService.proxy(
       'GET',
-      '/catalog/schemas',
+      `/catalog/schemas`,
       user,
       undefined,
       query,
@@ -763,18 +748,18 @@ export class PlatformApiController {
   }
 
   @Post('pipelines/catalog/tables/validate')
-  @ApiOperation({ summary: 'Validate tables and schemas' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
+  @ApiOperation({ summary: 'Validate Table and Schema' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async validateTableAndSchema(
+    @Body() payload: ValidationTableDTO,
     @User() user: RequestUser,
     @Query() query: Record<string, string>,
-    @Body() validateTablesDto: ValidateTablesDTO[]
   ) {
     return this.platformApiService.proxy(
       'POST',
-      '/catalog/tables/validate',
+      `/catalog/tables/validate`,
       user,
-      validateTablesDto,
+      payload,
       query,
     );
   }

src/modules/platform-api/platform-api.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ValidationTableDTO {
+    @ApiProperty()
+    tables: Array<{
+        table_name: string;
+        table_schema: string;
+    }>
+}

src/modules/storage-explorer/storage-explorer.service.ts

@@ -113,7 +113,7 @@ export class StorageExplorerService {
     }
 
     // Get customer-specific storage-explorer URL
-    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
+    const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id);
     const url = new URL(`${baseUrl}${path}`);
 
     // Add query params