Merge pull request #469 from dadosfera/feat/override-menu2

Feat/override menu2

README.md

@@ -4,7 +4,6 @@
 
 # Maestro
 
-
 Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
 
 ```mermaid

package-lock.json

@@ -17,7 +17,7 @@
         "@aws-sdk/signature-v4": "^3.370.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
         "@dadosfera/protospack": "2.5.3",
-        "@dadosfera/protospack-v2": "3.38.0-beta.28",
+        "@dadosfera/protospack-v2": "^3.39.0",
         "@grpc/grpc-js": "^1.9.3",
         "@grpc/proto-loader": "^0.7.9",
         "@nestjs/cli": "^9.5.0",
@@ -47,6 +47,7 @@
         "jwk-to-pem": "^2.0.5",
         "mixpanel": "^0.17.0",
         "ms": "^3.0.0-canary.1",
+        "multer": "^2.0.2",
         "openid-client": "^5.7.1",
         "passport": "^0.6.0",
         "passport-facebook": "^3.0.0",
@@ -1744,10 +1745,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.38.0-beta.28",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz",
-      "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==",
-      "license": "ISC",
+      "version": "3.39.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.39.0.tgz",
+      "integrity": "sha512-z6q+ZRpbDT6apfmly1+X818DgAFUCilFO0ATP7xR5+SH+GAUAJLY1dJRQDEkq5vFIfXnC2Nmm9aODZAs+9/5vw==",
       "dependencies": {
         "@grpc/grpc-js": "^1.9.3",
         "rxjs": "^7.5.5"
@@ -2928,6 +2928,20 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/@nestjs/platform-express/node_modules/concat-stream": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
+      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
+      "engines": [
+        "node >= 0.8"
+      ],
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^2.2.2",
+        "typedarray": "^0.0.6"
+      }
+    },
     "node_modules/@nestjs/platform-express/node_modules/content-disposition": {
       "version": "0.5.4",
       "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -3097,6 +3111,24 @@
       "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
       "license": "MIT"
     },
+    "node_modules/@nestjs/platform-express/node_modules/multer": {
+      "version": "1.4.4-lts.1",
+      "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.4-lts.1.tgz",
+      "integrity": "sha512-WeSGziVj6+Z2/MwQo3GvqzgR+9Uc+qt8SwHKh3gvNPiISKfsMfG4SvCOFYlxxgkXt7yIV2i1yczehm0EOKIxIg==",
+      "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.",
+      "dependencies": {
+        "append-field": "^1.0.0",
+        "busboy": "^1.0.0",
+        "concat-stream": "^1.5.2",
+        "mkdirp": "^0.5.4",
+        "object-assign": "^4.1.1",
+        "type-is": "^1.6.4",
+        "xtend": "^4.0.0"
+      },
+      "engines": {
+        "node": ">= 6.0.0"
+      }
+    },
     "node_modules/@nestjs/platform-express/node_modules/negotiator": {
       "version": "0.6.3",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
@@ -3121,6 +3153,25 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/@nestjs/platform-express/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/@nestjs/platform-express/node_modules/readable-stream/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
     "node_modules/@nestjs/platform-express/node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -3195,6 +3246,19 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/@nestjs/platform-express/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
+    "node_modules/@nestjs/platform-express/node_modules/string_decoder/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
+    },
     "node_modules/@nestjs/platform-express/node_modules/tslib": {
       "version": "2.5.3",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
@@ -6509,7 +6573,6 @@
       "engines": [
         "node >= 6.0"
       ],
-      "license": "MIT",
       "dependencies": {
         "buffer-from": "^1.0.0",
         "inherits": "^2.0.3",
@@ -9128,6 +9191,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
+    },
     "node_modules/isexe": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
@@ -10725,7 +10793,6 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/multer/-/multer-2.0.2.tgz",
       "integrity": "sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==",
-      "license": "MIT",
       "dependencies": {
         "append-field": "^1.0.0",
         "busboy": "^1.6.0",
@@ -11728,6 +11795,11 @@
         "url": "https://github.com/chalk/ansi-styles?sponsor=1"
       }
     },
+    "node_modules/process-nextick-args": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
+    },
     "node_modules/process-warning": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-1.0.0.tgz",

package.json

@@ -35,7 +35,7 @@
     "@aws-sdk/signature-v4": "^3.370.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
     "@dadosfera/protospack": "2.5.3",
-    "@dadosfera/protospack-v2": "3.38.0-beta.28",
+    "@dadosfera/protospack-v2": "^3.39.0",
     "@grpc/grpc-js": "^1.9.3",
     "@grpc/proto-loader": "^0.7.9",
     "@nestjs/cli": "^9.5.0",
@@ -65,6 +65,7 @@
     "jwk-to-pem": "^2.0.5",
     "mixpanel": "^0.17.0",
     "ms": "^3.0.0-canary.1",
+    "multer": "^2.0.2",
     "openid-client": "^5.7.1",
     "passport": "^0.6.0",
     "passport-facebook": "^3.0.0",
@@ -80,7 +81,6 @@
     "swagger-ui-express": "^4.6.3"
   },
   "overrides": {
-    "multer": "2.0.2",
     "form-data": "^4.0.4",
     "body-parser": "^1.20.3",
     "cross-spawn": "^7.0.5",

src/main.ts

@@ -111,4 +111,3 @@ function configureSwagger(app: INestApplication) {
   );
 }
 bootstrap();
-

src/modules/auth/auth.controller.ts

@@ -478,7 +478,6 @@ export class AuthController {
   @Get('me')
   async getMe(@Req() req: Request, @Res() res: Response) {
     this.logger.info('GET /auth/me ')
-    this.logger.info(JSON.stringify(req.headers));
 
     // Check for API key header first
     const apiKey = req.get('X-Api-key');

src/modules/catalog/catalog.service.ts

@@ -734,64 +734,6 @@ class CatalogService implements OnModuleInit {
     }
   }
 
-  async renameTableOnNimbus(
-    nimbusUrl: string,
-    nimbusId: number,
-    changes: { table_name?: string; table_schema?: string; display_name?: string },
-  ): Promise<void> {
-    const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`;
-    this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes });
-    await axios.patch(endpoint, changes);
-  }
-
-  async renameColumnMetadataOnNimbus(
-    nimbusUrl: string,
-    databaseName: string,
-    oldTableName: string,
-    oldTableSchema: string,
-    newTableName: string,
-    newTableSchema: string,
-  ): Promise<void> {
-    const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
-    this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint });
-    const { data: columns } = await axios.get(listEndpoint);
-
-    const filtered = Array.isArray(columns) ? columns : [];
-
-    for (const column of filtered) {
-      const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`;
-      await axios.patch(patchEndpoint, {
-        table_name: newTableName,
-        table_schema: newTableSchema,
-      });
-    }
-    this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`);
-  }
-
-  async renameDataPreviewOnNimbus(
-    nimbusUrl: string,
-    databaseName: string,
-    oldTableName: string,
-    oldTableSchema: string,
-    newTableName: string,
-    newTableSchema: string,
-  ): Promise<void> {
-    const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`;
-    this.logger.info(`Fetching data-preview records to rename`, { listEndpoint });
-    const { data: previews } = await axios.get(listEndpoint);
-
-    const filtered = Array.isArray(previews) ? previews : [];
-
-    for (const preview of filtered) {
-      const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`;
-      await axios.patch(patchEndpoint, {
-        table_name: newTableName,
-        table_schema: newTableSchema,
-      });
-    }
-    this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`);
-  }
-
   async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
     const customer_name_raw = metadata.get('customer_name');
 

src/modules/customers/customers.service.ts

@@ -9,12 +9,12 @@ import {
 } from '@nestjs/common';
 
 import { firstValueFrom, lastValueFrom } from 'rxjs';
-import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import { DucClient } from '../duc/client.config';
 import { ClientGrpc } from '@nestjs/microservices';
 import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
-import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { CustomerSetLinksRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
 import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
+import { CustomerLinksConfig } from './dtos/customers';
 import ErrorCodes from 'src/utils/errorCodes';
 import jwt from 'jsonwebtoken';
 import {
@@ -67,12 +67,12 @@ export class CustomersService implements OnModuleInit {
     )
   }
 
-  async getLinks(customerId: string) {
+  async getLinks(customerId: string): Promise<CustomerLinksConfig | null> {
     try {
       const result = await lastValueFrom(
-        this.customerService.CustomerFindOneById({ id: customerId }),
+        this.customerService.CustomerGetLinks({ customerId }),
       );
-      return result.customer?.links || [];
+      return (result.links as CustomerLinksConfig) || null;
     } catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
         throw new HttpException(err.details, HttpStatus.NOT_FOUND);
@@ -80,17 +80,17 @@ export class CustomersService implements OnModuleInit {
     }
   }
 
-  async setLinks(customerId: string, links: Link[]) {
+  async setLinks(customerId: string, links: CustomerLinksConfig) {
     if (!customerId || !links) {
       throw new HttpException(null, HttpStatus.BAD_REQUEST);
     }
 
     try {
       return await firstValueFrom(
-        this.customerService.CustomerUpdate({
-          id: customerId,
-          links,
-        } as CustomerUpdateRequest),
+        this.customerService.CustomerSetLinks({
+          customerId,
+          links: links as CustomerSetLinksRequest['links'],
+        }),
       );
     } catch (err) {
       if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)

src/modules/customers/dtos/customers.ts

@@ -1,7 +1,6 @@
-import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
 import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
-export class CustomerLink implements Link {
+export class CustomerLinkItem {
   @ApiProperty()
   href: string;
   @ApiProperty()
@@ -9,15 +8,59 @@ export class CustomerLink implements Link {
   @ApiProperty()
   description: string;
   @ApiPropertyOptional()
-  iconSrc: string;
+  iconSrc?: string;
 }
+
+export class CustomerSidebarLinkItem {
+  @ApiProperty()
+  type: 'link';
+  @ApiProperty({ type: Object })
+  title: Record<string, string>;
+  @ApiProperty()
+  link: string;
+  @ApiPropertyOptional()
+  icon?: string;
+}
+
+export class CustomerSidebarMenuItem {
+  @ApiProperty()
+  type: 'menu';
+  @ApiProperty({ type: Object })
+  title: Record<string, string>;
+  @ApiPropertyOptional()
+  icon?: string;
+  @ApiProperty({ type: [CustomerSidebarLinkItem] })
+  items: CustomerSidebarLinkItem[];
+}
+
+export class CustomerSidebarSection {
+  @ApiProperty({ type: Object })
+  title: Record<string, string>;
+  @ApiProperty({
+    type: 'array',
+    items: {
+      oneOf: [
+        { $ref: '#/components/schemas/CustomerSidebarMenuItem' },
+        { $ref: '#/components/schemas/CustomerSidebarLinkItem' },
+      ],
+    },
+  })
+  items: (CustomerSidebarMenuItem | CustomerSidebarLinkItem)[];
+}
+
+export class CustomerLinksConfig {
+  @ApiPropertyOptional({ type: [CustomerLinkItem] })
+  home?: CustomerLinkItem[];
+  @ApiPropertyOptional({ type: [CustomerSidebarSection] })
+  sidebar?: CustomerSidebarSection[];
+}
+
 export class CustomerLinkRequest {
-  @ApiProperty({ type: [CustomerLink] })
-  links: CustomerLink[];
+  @ApiProperty({ type: CustomerLinksConfig })
+  links: CustomerLinksConfig;
 }
 
 export class CustomerLinksResponse {
-  @ApiProperty({ type: [CustomerLink] })
-  links: CustomerLink[];
-}
-
+  @ApiPropertyOptional({ type: CustomerLinksConfig })
+  links?: CustomerLinksConfig;
+}

src/modules/inputs/dtos/input.model.ts

@@ -11,17 +11,19 @@ export class TableColumns {
   name: string;
   @ApiProperty()
   columns: string[];
-  @ApiProperty()
+  @ApiPropertyOptional({ type: [Column] })
   references: Column[];
   @ApiProperty()
-  identifier_columns: string[];
-  @ApiProperty()
   destination: Record<'raw' | 'qualify', {
     table_name: string;
     table_schema: string;
   }> | null;
   @ApiProperty()
   type: string;
+  @ApiPropertyOptional({ type: [String] })
+  identifier_columns?: string[];
+  @ApiPropertyOptional({ type: Column })
+  reference_column?: Column;
 }
 export class AvailableEntity {
   @ApiProperty()

src/modules/inputs/inputs.service.ts

@@ -164,6 +164,11 @@ export class InputsService {
     const inputCreateGenericRequest: InputCreateGenericRequest = {
       input: {
         ...body,
+        tables: (body.tables || []).map((table) => ({
+          ...table,
+          identifier_columns: table.identifier_columns || [],
+          reference_column: table.reference_column || table.references?.[0],
+        })),
       },
       info,
     };
@@ -200,7 +205,7 @@ export class InputsService {
   }
 
   async update(id: string, data, info: Info) {
-    // this.validateCron({ ...data, info });
+    this.validateCron({ ...data, info });
     try {
       const updateInputResponse: any = await this.OLD_inputClient.update({
         id,

src/modules/pipelinesV2/interfaces.ts

@@ -1,15 +1,6 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
 import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
 
-export class PipelineInputsDTO {
-  @ApiProperty()
-  tables: Array<{
-    name: string,
-    type: string,
-    
-  }>
-}
-
 export class IPipelineV2 {
   @ApiProperty()
   id: string;
@@ -132,30 +123,3 @@ export class PipelineFindAllReq {
   @ApiPropertyOptional()
   type?: string | undefined;
 }
-
-export interface UpdateTableDTO {
-  name: string;
-  type: string;
-  columns: string[];
-  destinations: {
-    raw: {
-      table_schema: string;
-      table_name: string;
-    };
-    qualify: {
-      table_schema: string;
-      table_name: string;
-    };
-  };
-  identifier_columns: string[];
-  reference_column: {
-    name: string;
-    type: string;
-  };
-  memory: number;
-}
-
-export interface UpdatePlatformInputRequest {
-  cron: string;
-  tables: Array<UpdateTableDTO>;
-}

src/modules/pipelinesV2/pipelines.controller.ts

@@ -43,15 +43,11 @@ import {
   IPipelineV2,
   IInitUploadCSVFile,
   PipelineFindAllReq,
-  UpdatePlatformInputRequest,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { Language } from 'src/decorators/language.decorator';
 import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
-import { TableColumns } from '../inputs/dtos/input.model';
-import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
-import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 
 @ApiTags('PipelinesV2')
 @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -227,7 +223,6 @@ export class PipelinesController {
       .then((res) => {
         //{pipeline:{tables: {tables: [], input_id: ''}}}
         let tables = JSON.parse(res.pipeline.config.tables);
-        const input_id = tables?.input_id;
         if (tables?.tables) tables = tables.tables;
         Object.assign(res.pipeline, {
           transformations: res.pipeline.transformations
@@ -236,7 +231,6 @@ export class PipelinesController {
           config: {
             cron: res.pipeline.config.cron,
             tables,
-            input_id
           },
           properties: res.pipeline.properties
             ? JSON.parse(res.pipeline.properties)
@@ -283,45 +277,6 @@ export class PipelinesController {
     return response;
   }
 
-  @Patch('/:pipelineId/inputs/:id')
-  @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
-  async updatePipelineInput(
-    @Language() language: LanguageEnum,
-    @Body() pipelineInputDTO: UpdatePlatformInputRequest,
-    @Param('id') inputId: string,
-    @Param('pipelineId') pipelineId: string,
-    @User() user: RequestUser,
-  ) {
-    this.logger.info('PipelinesController - update', { user });
-
-
-    const { customer_id, customer_name, user_id, username } = user;
-    const info: Info = {
-      user_id: user.user_id,
-      customer: user.customer_name,
-      customer_id: user.customer_id,
-    };
-    const metadata = PackTheMetadata({
-      customer_id,
-      customer_name,
-      user_id,
-      username,
-      language,
-    });
-
-    const response = await this.pipelinesClientService.updatePipelineInput(
-      pipelineId,
-      inputId,
-      pipelineInputDTO,
-      info,
-      user,
-      metadata,
-    );
-
-    this.logger.info('PipelinesController - update: OK', { user });
-    return response;
-  }
-
   @ApiInternalOnlyEndpoint()
   @Put('/:id')
   @ApiOperation({

src/modules/pipelinesV2/pipelines.module.ts

@@ -11,7 +11,6 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe
 import { ConnectorModule } from '../connector/connector.module';
 import { InputsModule } from '../inputs/inputs.module';
 import { TransformationsModule } from '../transformations/transformations.module';
-import { PlatformApiModule } from '../platform-api/platform-api.module';
 
 const client = new PipelinesClientConfiguration();
 
@@ -22,7 +21,6 @@ const client = new PipelinesClientConfiguration();
     ConnectorModule,
     InputsModule,
     TransformationsModule,
-    PlatformApiModule
   ],
   controllers: [PipelinesController],
   providers: [PipelinesService, DadosferaLogger],

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,4 +1,3 @@
-/* eslint-disable no-async-promise-executor */
 import {
   BadRequestException,
   HttpException,
@@ -17,7 +16,7 @@ import { lastValueFrom } from 'rxjs';
 
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { PipelinesClientConfiguration } from './pipelines-client';
-import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
+import { ICreatePipelineV2Req } from './interfaces';
 import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
@@ -27,8 +26,6 @@ import { TransformationsService } from '../transformations/transformations.servi
 import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
 import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
-import { PlatformApiService } from '../platform-api/platform-api.service';
-import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -42,7 +39,6 @@ export class PipelinesService implements OnModuleInit {
     private readonly connectorService: ConnectorClientService,
     private readonly inputsService: InputsService,
     private readonly transformationsService: TransformationsService,
-    private readonly platformAPI: PlatformApiService
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -142,7 +138,6 @@ export class PipelinesService implements OnModuleInit {
     const findOnePipelineResponse = await lastValueFrom(
       this.pipelineReadService.PipelineV2FindOne(data, metadata),
     );
-    console.log('pipeline find one response', findOnePipelineResponse);
     this.logger.info('Done');
 
     return findOnePipelineResponse;
@@ -344,137 +339,4 @@ export class PipelinesService implements OnModuleInit {
 
     return res;
   }
-
-  async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
-    this.logger.info('InputClientService - Update');
-
-    this.logger.info('Update Dynamo Reference');
-    const pipelineIdFormat = pipelineId.split('-').join('_');
-    const updateInputResponse = await this.inputsService.update(
-      inputId,
-      updateInputDTO,
-      info
-    )
-
-    const requests = [];
-
-    this.logger.info('Dynamo Response', updateInputResponse);
-
-    for (const [index, table] of updateInputDTO.tables.entries()) {
-      const id = `${pipelineIdFormat}_${index}`;
-      this.logger.info('Updating input reference for table', table.name);
-      const body = {}
-      if (table.columns) {
-        body['column_include_list'] = table.columns;
-      }
-
-      if (table.reference_column) {
-        body['incremental_column_name'] = table.reference_column.name;
-        body['incremental_column_type'] = table.reference_column.type;
-      }
-
-      if (table.identifier_columns) {
-        body['primary_keys'] = table.identifier_columns;
-      }
-
-      this.logger.info('Request body', body);
-      const updateCollumns = this.platformAPI.proxy(
-        'PATCH',
-        `/jobs/${id}/input`,
-        user,
-        body
-      )
-      requests.push(updateCollumns);
-
-      if (table.memory) {
-        this.logger.info('Updating memory allocation for table', table.name);
-        const updateMemory = this.platformAPI.proxy(
-          'PUT',
-          `/jobs/${id}/memory`,
-          user,
-          {
-            amount: table.memory
-          }
-        )
-        requests.push(updateMemory);
-      }
-
-      if (table.type) {
-          const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
-          requests.push(updateSyncMode);
-      }
-    }
-
-    this.logger.info('Create Platform Request for each JOB');
-
-    if (updateInputDTO.cron) {
-      const crnUpdatedRequest = new Promise(async (resolve, reject) => {
-        const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
-
-        if (response.error) {
-          this.logger.error('Error updating pipeline cron', response.error);
-          return reject(new ErrorBuilder(response.error));
-        }
-        this.logger.error('Pipeline cron updated successfully', response);
-        return resolve(response);
-      });
-      requests.push(crnUpdatedRequest);
-    }
-
-    this.logger.info('Executing all request for the platform api');
-
-    const results = await Promise.allSettled(requests);
-    this.logger.info('Platform api response', results);
-
-    return updateInputResponse;
-    
-  }
-
-  private async updatePipelineSyncMode(table: UpdateTableDTO,  pipelineId: string, user: RequestUser) {
-    const body = {
-      target_load_type: table.type
-    }
-
-    if (table.type === 'incremental_with_qualify') {
-      body['incremental_column_name'] = table.reference_column.name;
-      body['incremental_column_type'] = table.reference_column.type;
-      body['primary_keys'] = table.identifier_columns;
-    }
-
-    if (table.type === 'incremental') {
-      body['incremental_column_name'] = table.reference_column.name;
-      body['incremental_column_type'] = table.reference_column.type;
-    }
-
-    this.logger.info('Updating pipeline sync mode', {
-      pipelineId,
-      body
-    });
-
-    return this.platformAPI.proxy(
-      "POST",
-      `/jobs/jdbc/${pipelineId}/sync-mode`,
-      user,
-      body
-    )
-  }
-
-  private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
-    try {
-      const response = await this.platformAPI.proxy(
-        'PATCH',
-        `/pipeline/${pipelineId}`,
-        user,
-        {
-          cron
-        }
-      );
-
-      return response
-    } catch (error) {
-        return {
-          error: error.message
-        }
-    }
-  }
 }

src/modules/platform-api/platform-api.controller.ts

@@ -10,7 +10,6 @@ import {
   Query,
   Inject,
   BadRequestException,
-  HttpException,
 } from '@nestjs/common';
 import { ApiTags, ApiOperation } from '@nestjs/swagger';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -26,9 +25,6 @@ import { ElasticsearchService } from '../../services/elasticsearch';
 import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
 import { CustomersService } from '../customers/customers.service';
 import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
-import { CatalogService } from '../catalog/catalog.service';
-import { PackTheMetadata } from '../../utils/PackTheMetadata';
-import { ValidationTableDTO } from './platform-api.dto';
 
 
 type ValidateTablesDTO = {
@@ -38,11 +34,6 @@ type ValidateTablesDTO = {
   }>
 }
 
-type RenameTablesBody = {
-  raw?: { table_name: string; table_schema: string };
-  qualify?: { table_name: string; table_schema: string };
-}
-
 @ApiTags('Platform API')
 @Controller('platform')
 export class PlatformApiController {
@@ -53,7 +44,6 @@ export class PlatformApiController {
     private readonly elasticsearchService: ElasticsearchService,
     private readonly dynamoDBService: DynamoDBService,
     private readonly customersService: CustomersService,
-    private readonly catalogService: CatalogService,
     @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
   ) {
     this.logger = dadosferaLogger.logger;
@@ -85,23 +75,6 @@ export class PlatformApiController {
     return jobId?.replace(/-/g, '_') || '';
   }
 
-  private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise<any> {
-    const connectorTypes = ['jdbc', 'singer', 's3'];
-    for (const type of connectorTypes) {
-      try {
-        const job = await this.platformApiService.proxy(
-          'GET',
-          `/jobs/${type}/${normalizedJobId}`,
-          user,
-        );
-        return job;
-      } catch (error) {
-        // Continue to next connector type
-      }
-    }
-    throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404);
-  }
-
   /**
    * Extract the pipeline ID (base UUID) from a job ID.
    * Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1").
@@ -755,10 +728,26 @@ export class PlatformApiController {
     );
   }
 
-  // ==================== PIPELINE VALIDATION ====================
+  // ==================== Catalog ROUTES ====================
+
+  @Get('pipelines/catalog/tables')
+  @ApiOperation({ summary: 'Get all tables available' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  async getAvailableTables(
+    @User() user: RequestUser,
+    @Query() query: Record<string, string>,
+  ) {
+    return this.platformApiService.proxy(
+      'GET',
+      '/catalog/tables',
+      user,
+      undefined,
+      query,
+    );
+  }
 
   @Get('pipelines/catalog/schemas')
-  @ApiOperation({ summary: 'Get available schemas' })
+  @ApiOperation({ summary: 'Get all schemas available' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
   async getAvailableSchemas(
     @User() user: RequestUser,
@@ -766,7 +755,7 @@ export class PlatformApiController {
   ) {
     return this.platformApiService.proxy(
       'GET',
-      `/catalog/schemas`,
+      '/catalog/schemas',
       user,
       undefined,
       query,
@@ -774,18 +763,18 @@ export class PlatformApiController {
   }
 
   @Post('pipelines/catalog/tables/validate')
-  @ApiOperation({ summary: 'Validate Table and Schema' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
+  @ApiOperation({ summary: 'Validate tables and schemas' })
+  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
   async validateTableAndSchema(
-    @Body() payload: ValidationTableDTO,
     @User() user: RequestUser,
     @Query() query: Record<string, string>,
+    @Body() validateTablesDto: ValidateTablesDTO[]
   ) {
     return this.platformApiService.proxy(
       'POST',
-      `/catalog/tables/validate`,
+      '/catalog/tables/validate',
       user,
-      payload,
+      validateTablesDto,
       query,
     );
   }
@@ -979,192 +968,6 @@ export class PlatformApiController {
     return result;
   }
 
-  @Post('jobs/:jobId/rename-tables')
-  @ApiOperation({ summary: 'Rename job output tables and sync to catalog' })
-  @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
-  async renameJobTables(
-    @Param('jobId') jobId: string,
-    @Body() body: RenameTablesBody,
-    @User() user: RequestUser,
-  ) {
-    const normalizedJobId = this.normalizeJobId(jobId);
-
-    const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user);
-
-    const result = await this.platformApiService.proxy(
-      'POST',
-      `/jobs/${normalizedJobId}/rename-tables`,
-      user,
-      body,
-    );
-
-    try {
-      await this.syncTableRenameToCatalog(jobId, body, currentJob, user);
-    } catch (error) {
-      this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message });
-
-      const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {});
-      if (reverseBody) {
-        try {
-          await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody);
-          this.logger.info('Snowflake rename rolled back', { jobId });
-        } catch (rollbackError) {
-          this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message });
-        }
-      }
-
-      throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500);
-    }
-
-    return result;
-  }
-
-  private buildSnowflakeRollbackBody(
-    body: RenameTablesBody,
-    outputConfig: any,
-  ): RenameTablesBody | null {
-    const reverse: RenameTablesBody = {};
-
-    if (body.raw) {
-      const nested = outputConfig.raw;
-      const oldTableName = nested?.table_name || outputConfig.table_name;
-      const oldTableSchema = nested?.table_schema || 'PUBLIC';
-      if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema };
-    }
-
-    if (body.qualify) {
-      const nested = outputConfig.qualify;
-      if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' };
-    }
-
-    return Object.keys(reverse).length > 0 ? reverse : null;
-  }
-
-  /**
-   * Sync table rename to Elasticsearch and Nimbus.
-   *
-   * For each target (raw, qualify):
-   *  1. Resolve old table name from output_config
-   *  2. Find the ES data asset by pipeline + table + schema
-   *  3. Update ES, Nimbus table-metadata, column-metadata, and data-preview
-   *  4. If any step fails, rollback all completed steps for that target
-   */
-  private async syncTableRenameToCatalog(
-    jobId: string,
-    body: RenameTablesBody,
-    currentJob: any,
-    user: RequestUser,
-  ): Promise<void> {
-    const pipelineId = this.extractPipelineIdFromJobId(jobId);
-    const outputConfig = currentJob.output_config || {};
-    const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } });
-    const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`;
-
-    const targets = this.buildRenameTargets(body, outputConfig);
-
-    for (const { key, oldTableName, oldTableSchema, newValues } of targets) {
-      const rollbackSteps: Array<() => Promise<void>> = [];
-
-      try {
-        const dataAsset = await this.elasticsearchService.findDataAssetByTable(
-          user.customer_name, oldTableName, oldTableSchema,
-        );
-
-        if (!dataAsset) {
-          this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema });
-          continue;
-        }
-
-        const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset;
-        const oldValues = { table_name: oldTableName, table_schema: oldTableSchema };
-
-        // ES update
-        const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name };
-        await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields);
-        rollbackSteps.push(() => this.elasticsearchService.updateDataAsset(
-          user.customer_name, esAssetId,
-          { name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName },
-        ));
-
-        const newTableNameUpper = newValues.table_name.toUpperCase();
-        const newTableSchemaUpper = newValues.table_schema.toUpperCase();
-        const oldTableNameUpper = oldTableName.toUpperCase();
-        const oldTableSchemaUpper = oldTableSchema.toUpperCase();
-
-        // Nimbus table-metadata
-        if (nimbusId) {
-          await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper });
-          rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper }));
-        }
-
-        // Nimbus column-metadata
-        await this.catalogService.renameColumnMetadataOnNimbus(
-          nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
-        );
-        rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus(
-          nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
-        ));
-
-        // Nimbus data-preview
-        await this.catalogService.renameDataPreviewOnNimbus(
-          nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
-        );
-        rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus(
-          nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
-        ));
-
-        this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name });
-      } catch (error) {
-        this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message });
-        await this.executeRollback(rollbackSteps, key, jobId);
-        throw error;
-      }
-    }
-  }
-
-  private buildRenameTargets(
-    body: RenameTablesBody,
-    outputConfig: any,
-  ): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> {
-    const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' };
-    const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = [];
-
-    for (const key of ['raw', 'qualify'] as const) {
-      if (!body[key]) continue;
-
-      const nested = outputConfig[key];
-
-      // qualify: only sync if output_config.qualify already exists
-      if (key === 'qualify' && !nested?.table_name) continue;
-
-      const oldTableName = nested?.table_name || outputConfig.table_name;
-      if (!oldTableName) continue;
-
-      targets.push({
-        key,
-        oldTableName,
-        oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key],
-        newValues: body[key],
-      });
-    }
-
-    return targets;
-  }
-
-  private async executeRollback(
-    steps: Array<() => Promise<void>>,
-    targetKey: string,
-    jobId: string,
-  ): Promise<void> {
-    for (const rollback of steps.reverse()) {
-      try {
-        await rollback();
-      } catch (error) {
-        this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message });
-      }
-    }
-  }
-
   @Get('jobs/jdbc/configs/allowed_datatypes')
   @ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
   @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)

src/modules/platform-api/platform-api.dto.ts

@@ -1,9 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-
-export class ValidationTableDTO {
-    @ApiProperty()
-    tables: Array<{
-        table_name: string;
-        table_schema: string;
-    }>
-}

src/modules/platform-api/platform-api.module.ts

@@ -7,10 +7,9 @@ import { PlatformApiService } from './platform-api.service';
 import { ElasticsearchModule } from '../../services/elasticsearch';
 import { DynamoDBModule } from '../../services/dynamodb';
 import { CustomersModule } from '../customers/customers.module';
-import { CatalogModule } from '../catalog/catalog.module';
 
 @Module({
-  imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule],
+  imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
   controllers: [PlatformApiController],
   providers: [PlatformApiService, DadosferaLogger],
   exports: [PlatformApiService],

src/modules/platform-api/platform-api.service.ts

@@ -89,12 +89,6 @@ export class PlatformApiService {
 
       // Propagate non-2xx responses as HttpExceptions
       if (response.status >= 400) {
-        this.logger.error('Platform API upstream error', {
-          status: response.status,
-          data: response.data,
-          path,
-          method: method.toUpperCase(),
-        });
         throw new HttpException(response.data, response.status);
       }
 

src/services/elasticsearch/elasticsearch.service.ts

@@ -358,81 +358,6 @@ export class ElasticsearchService {
     }
   }
 
-  private getDataAssetIndex(customerName: string): string {
-    return `${customerName}_data_assets_catalog`;
-  }
-
-  async findDataAssetByTable(
-    customerName: string,
-    tableName: string,
-    tableSchema: string,
-  ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> {
-    const index = this.getDataAssetIndex(customerName);
-
-    this.logger.info('Elasticsearch: Searching data asset', {
-      index,
-      tableName,
-      tableSchema,
-    });
-
-    try {
-      const response = await this.client.post(`/${index}/_search`, {
-        query: {
-          bool: {
-            must: [
-              { term: { 'table_name.keyword': tableName.toUpperCase() } },
-              { term: { 'table_schema.keyword': tableSchema.toUpperCase() } },
-            ],
-          },
-        },
-        size: 1,
-      });
-
-      const hits = response.data.hits?.hits || [];
-      if (hits.length === 0) {
-        this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index });
-        return null;
-      }
-
-      return { ...hits[0]._source, _es_id: hits[0]._id };
-    } catch (error) {
-      this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index });
-      throw error;
-    }
-  }
-
-  async updateDataAsset(
-    customerName: string,
-    assetId: string,
-    updates: Record<string, any>,
-  ): Promise<any> {
-    const index = this.getDataAssetIndex(customerName);
-
-    this.logger.info('Elasticsearch: Updating data asset', {
-      index,
-      assetId,
-      fields: Object.keys(updates),
-    });
-
-    try {
-      const response = await this.client.post(
-        `/${index}/_update/${assetId}`,
-        { doc: updates },
-        { params: { refresh: 'wait_for' } },
-      );
-
-      this.logger.info('Elasticsearch: Data asset updated', {
-        assetId,
-        result: response.data.result,
-      });
-
-      return response.data;
-    } catch (error) {
-      this.handleError('updateDataAsset', error, { assetId, index });
-      throw error;
-    }
-  }
-
   private handleError(
     operation: string,
     error: any,