FIX: connections migration

src/modules/connection-migrations/connection-migrations.controller.ts

@@ -1,16 +0,0 @@
-import { Controller, Get } from '@nestjs/common';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { ConnectionMigrationsService } from './connection-migrations.service';
-
-@Controller('connection-migrations')
-export class ConnectionMigrationsController {
-  constructor(
-    private connectionMigrationsService: ConnectionMigrationsService,
-  ) {}
-  @Get()
-  @RequireAllPermissions(PERMISSIONS_GROUPS.DADOSFERA.permissions.ADMIN)
-  async getAllCustomersPlatformConnections() {
-    return this.connectionMigrationsService.synchronizeConnections();
-  }
-}

src/modules/connection-migrations/connection-migrations.module.ts

@@ -1,13 +0,0 @@
-import { Module } from '@nestjs/common';
-import { ConnectionMigrationsService } from './connection-migrations.service';
-import { ConnectionMigrationsController } from './connection-migrations.controller';
-import { UsersModule } from '../users/users.module';
-import { ConnectionModule } from '../connection/connection.module';
-import { ConnectorModule } from '../connector/connector.module';
-
-@Module({
-  imports: [UsersModule, ConnectionModule, ConnectorModule],
-  providers: [ConnectionMigrationsService],
-  controllers: [ConnectionMigrationsController],
-})
-export class ConnectionMigrationsModule {}

src/modules/connection-migrations/connection-migrations.service.ts

@@ -1,44 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { ConnectionClientService } from '../connection/client.service';
-import { ConnectorClientService } from '../connector/client.service';
-import { UsersService } from '../users/users.service';
-
-@Injectable()
-export class ConnectionMigrationsService {
-  constructor(
-    private usersService: UsersService,
-    private connectionsService: ConnectionClientService,
-    private connectorService: ConnectorClientService,
-  ) {}
-  async synchronizeConnections() {
-    const { connectors } = await this.connectorService
-      .getAllConnectors({
-        search: undefined,
-        filters: undefined,
-        language: 'pt-br',
-        page: 1,
-        size: 500,
-      })
-      .then(({ connectors }) => ({ connectors: JSON.parse(connectors) }));
-    const { customers } = await this.usersService.findaAllCustomers();
-    const promises: Promise<{
-      customer_name: string;
-      productConnections: Record<string, any>;
-      platformConnections: Record<string, any>;
-    }>[] = [];
-    customers.forEach((c) => {
-      const customer_name = c.name;
-      promises.push(
-        this.connectionsService
-          .INTERNAL_getAllConnections({ customer_name })
-          .then((res) => ({
-            customer_name,
-            platformConnections: JSON.parse(res.platformConnections),
-            productConnections: JSON.parse(res.productConnections),
-          })),
-      );
-    });
-    const customer_connections = await Promise.all(promises);
-    return { customer_connections, connectors };
-  }
-}

src/modules/connection/client.service.ts

@@ -8,8 +8,10 @@ import { ClientGrpc } from '@nestjs/microservices';
 import { ConnectionManager } from '@dadosfera/protospack-v2';
 import { lastValueFrom } from 'rxjs';
 import {
+  ConnectionApiConnection,
   ConnectionRes,
   ConnectionsRes,
+  ConnectionToCatalogDto,
   CreateConnectionDto,
   UpdateConnectionDto,
 } from './dtos/connection';
@@ -18,6 +20,8 @@ import {
   GetConnectionDetailsResponse,
   INTERNAL_GetAllConnectionsResponse,
 } from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
+import { UsersService } from '../users/users.service';
+import { ConnectorClientService } from '../connector/client.service';
 
 export class ConnectionClientService implements OnModuleInit {
   private connectionServiceRead: ConnectionManager.ReadService.ConnectionManagerReadServices;
@@ -25,6 +29,8 @@ export class ConnectionClientService implements OnModuleInit {
   constructor(
     @Inject(ConnectionClientConfiguration.name)
     private readonly grpcClient: ClientGrpc,
+    private usersService: UsersService,
+    private connectorService: ConnectorClientService,
   ) {}
 
   onModuleInit() {
@@ -163,4 +169,105 @@ export class ConnectionClientService implements OnModuleInit {
       this.connectionServiceRead.INTERNAL_GetAllConnections({ customer_name }),
     );
   }
+
+  async catalogExistingConnection(data: ConnectionToCatalogDto) {
+    return lastValueFrom(
+      this.connectionServiceRead.INTERNAL_CatalogExistingConnection({
+        data_str: JSON.stringify(data),
+      }),
+    );
+  }
+
+  async synchronizeConnections(apply: boolean) {
+    const connectors = await this.connectorService
+      .getAllConnectors({
+        search: undefined,
+        filters: undefined,
+        language: 'pt-br',
+        page: 1,
+        size: 500,
+      })
+      .then(({ connectors: connectorsStr }) =>
+        JSON.parse(connectorsStr).map((c) => c),
+      );
+    const { customers } = await this.usersService.findaAllCustomers();
+    const promises: Promise<{
+      customer_name: string;
+      customer_id: string;
+      productConnections: Record<string, any>;
+      platformConnections: ConnectionApiConnection[];
+    }>[] = [];
+    customers.forEach((c) => {
+      const customer_name = c.name;
+      const customer_id = c.id;
+      promises.push(
+        this.INTERNAL_getAllConnections({ customer_name }).then((res) => ({
+          customer_name,
+          customer_id,
+          platformConnections: JSON.parse(res.platformConnections),
+          productConnections: JSON.parse(res.productConnections),
+        })),
+      );
+    });
+    const customer_connections = await Promise.all(promises);
+
+    const missingInProduct: ConnectionApiConnection[] = [];
+    const migrations = customer_connections.map((cc) => {
+      const {
+        customer_name,
+        platformConnections,
+        productConnections,
+        customer_id,
+      } = cc;
+
+      platformConnections.forEach((plc) => {
+        const productConnection = productConnections.find(
+          (prc) => prc.id === plc.config_id,
+        );
+        if (!productConnection) missingInProduct.push(plc);
+      });
+
+      const updatePromises: Promise<any>[] = [];
+      for (const connection of missingInProduct) {
+        const connector = connectors.find(
+          (c) => c._source.plugin === connection.plugin,
+        );
+        const connectionToCatalog = {
+          id: connection.config_id,
+          created_at: connection.created_at,
+          updated_at: new Date().toISOString(),
+          customer_id: customer_id,
+          user_id: '',
+          plugin: connection.plugin,
+          name: connection.name,
+          description: connection.description,
+          image_url: connector._source.image,
+          network_config_id: connection.network_config_id,
+          type: connection.type,
+          connector_id: connector._id,
+          connector_name: connector._source.name,
+          in_use: 1,
+          username: 'auto',
+          customer_name: customer_name,
+        };
+        updatePromises.push(
+          this.catalogExistingConnection(connectionToCatalog),
+        );
+      }
+      return { customer: { ...cc }, updatePromises, missingInProduct };
+    });
+    const customersStatus: any[] = [];
+    for (const migration of migrations) {
+      let status: any[] = [];
+      if (apply) {
+        status = await Promise.all(migration.updatePromises);
+      }
+      customersStatus.push({
+        customer: migration.customer.customer_name,
+        status,
+        missingInProduct: migration.missingInProduct,
+      });
+    }
+    return { customersStatus };
+  }
 }

src/modules/connection/connection.controller.ts

@@ -44,6 +44,12 @@ export class ConnectionController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @Get('connections-migration')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.DADOSFERA.permissions.ADMIN)
+  async getAllCustomersPlatformConnections(@Query('apply') apply: string) {
+    return this.clientService.synchronizeConnections(apply === 'true');
+  }
+
   @Post()
   @RequireAllPermissions(connectionPermissions.CREATE)
   async createConnection(

src/modules/connection/connection.module.ts

@@ -4,13 +4,19 @@ import { ConnectionClientService } from './client.service';
 import { ConnectionClientConfiguration } from './client.config';
 import { ConnectionController } from './connection.controller';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { UsersModule } from '../users/users.module';
+import { ConnectorModule } from '../connector/connector.module';
 
 const client = new ConnectionClientConfiguration();
 
 @Module({
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    UsersModule,
+    ConnectorModule,
+  ],
   controllers: [ConnectionController],
   providers: [ConnectionClientService, DadosferaLogger],
-  imports: [ClientsModule.register([client.providerOptions])],
   exports: [ConnectionClientService],
 })
 export class ConnectionModule {}

src/modules/connection/dtos/connection.ts

@@ -13,7 +13,30 @@ import {
   GetConnectionDetailsResponse,
   GetConnectionResponse,
 } from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
-
+export type ConnectionTypes = 'database' | 'file' | 'application';
+export type ConnectionCredentialsType =
+  | 'basic_auth'
+  | 'iam_user'
+  | 'role_name'
+  | 'oauth'
+  | 'api_key'
+  | 'service_account'
+  | 'headers_authF';
+export interface ConnectionApiConnection {
+  config_id: string;
+  plugin: string;
+  secret_id: string;
+  created_at: string;
+  network_config_id: null;
+  type: ConnectionTypes;
+  credentials_type: ConnectionCredentialsType;
+  credentials: Record<string, any>;
+  sensitive_fields: string[];
+  name: string;
+  customer_name: string;
+  description: string;
+  updated_at: string;
+}
 export class ConnectionDto implements Connection {
   // ---Automatically generated information will not be sent by the frontend--- //
   @ApiProperty()
@@ -70,21 +93,17 @@ export class ConnectionDto implements Connection {
 }
 
 export class ConnectionToCatalogDto implements ConnectionToCatalog {
-  // ---Automatically generated information will not be sent by the frontend--- //
-  @ApiProperty()
+  // ---Automatically generated information - will not be sent by the frontend--- //
   id: string;
-
-  @ApiProperty()
   customer_id: string;
-
-  @ApiProperty()
   updated_at: string;
-
-  @ApiProperty()
   created_at: string;
+  // ---Information extracted from token--- //
+  user_id: string;
+  username: string;
+  customer_name: string;
 
   // ---Information sent by the frontend--- //
-
   @ApiProperty()
   name: string;
 
@@ -98,7 +117,6 @@ export class ConnectionToCatalogDto implements ConnectionToCatalog {
   network_config_id: string | null;
 
   // ---Information sent by the frontend (Heirs of Connector)--- //
-
   @ApiProperty()
   connector_id: string;