Merge pull request #132 from dadosfera/fix/connections-migration

Fix: connections migration

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
-        "@dadosfera/protospack-v2": "3.17.0",
+        "@dadosfera/protospack-v2": "3.17.2",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1726,9 +1726,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.17.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.17.0.tgz",
-      "integrity": "sha512-NeS8cs+ZJrC7kW1mgnRWwmP5E70RTfjp+UlJkHN5RA0vsRhKVcF8dA5VmGQerPT4aDA3K1sMkWJbjzU3Wdt/jA==",
+      "version": "3.17.2",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.17.2.tgz",
+      "integrity": "sha512-N1hn7lIBWgc4JPEAdlQ3jt+++y61XYb9amDmgOUbfSt4803oSdAnCktrcxq3/0e/ZuSYm0qESOkKn0uNLDlGcg==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12305,9 +12305,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.17.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.17.0.tgz",
-      "integrity": "sha512-NeS8cs+ZJrC7kW1mgnRWwmP5E70RTfjp+UlJkHN5RA0vsRhKVcF8dA5VmGQerPT4aDA3K1sMkWJbjzU3Wdt/jA==",
+      "version": "3.17.2",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.17.2.tgz",
+      "integrity": "sha512-N1hn7lIBWgc4JPEAdlQ3jt+++y61XYb9amDmgOUbfSt4803oSdAnCktrcxq3/0e/ZuSYm0qESOkKn0uNLDlGcg==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -27,7 +27,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
-    "@dadosfera/protospack-v2": "3.17.0",
+    "@dadosfera/protospack-v2": "3.17.2",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/app.module.ts

@@ -23,7 +23,6 @@ import { HealthModule } from './modules/health/health.module';
 import { CatalogModule } from './modules/catalog/catalog.module';
 import { ConfigModule } from '@nestjs/config';
 import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
-import { ConnectionMigrationsModule } from './modules/connection-migrations/connection-migrations.module';
 
 @Module({
   controllers: [],
@@ -54,7 +53,6 @@ import { ConnectionMigrationsModule } from './modules/connection-migrations/conn
     InputsModule,
     OauthModule,
     PipelinesV2Module,
-    ConnectionMigrationsModule,
   ],
 })
 export class AppModule {}

src/authentication/permissions.enum.ts

@@ -505,12 +505,12 @@ export const PERMISSIONS_GROUPS = {
     permissions: {
       ADMIN: {
         seqid: 40,
-        claim: 'dadosfera:connections-migation',
+        claim: 'dadosfera:connections-migration',
         usage: PermissionUsages.INTERNAL,
         name: {
-          'pt-br': 'connections-migation',
-          'en-us': 'connections-migation',
-          'es-es': 'connections-migation',
+          'pt-br': 'connections-migration',
+          'en-us': 'connections-migration',
+          'es-es': 'connections-migration',
         },
       },
     },

src/modules/connection-migrations/connection-migrations.controller.ts

@@ -1,16 +0,0 @@
-import { Controller, Get } from '@nestjs/common';
-import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
-import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
-import { ConnectionMigrationsService } from './connection-migrations.service';
-
-@Controller('connection-migrations')
-export class ConnectionMigrationsController {
-  constructor(
-    private connectionMigrationsService: ConnectionMigrationsService,
-  ) {}
-  @Get()
-  @RequireAllPermissions(PERMISSIONS_GROUPS.DADOSFERA.permissions.ADMIN)
-  async getAllCustomersPlatformConnections() {
-    return this.connectionMigrationsService.synchronizeConnections();
-  }
-}

src/modules/connection-migrations/connection-migrations.module.ts

@@ -1,13 +0,0 @@
-import { Module } from '@nestjs/common';
-import { ConnectionMigrationsService } from './connection-migrations.service';
-import { ConnectionMigrationsController } from './connection-migrations.controller';
-import { UsersModule } from '../users/users.module';
-import { ConnectionModule } from '../connection/connection.module';
-import { ConnectorModule } from '../connector/connector.module';
-
-@Module({
-  imports: [UsersModule, ConnectionModule, ConnectorModule],
-  providers: [ConnectionMigrationsService],
-  controllers: [ConnectionMigrationsController],
-})
-export class ConnectionMigrationsModule {}

src/modules/connection-migrations/connection-migrations.service.ts

@@ -1,44 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { ConnectionClientService } from '../connection/client.service';
-import { ConnectorClientService } from '../connector/client.service';
-import { UsersService } from '../users/users.service';
-
-@Injectable()
-export class ConnectionMigrationsService {
-  constructor(
-    private usersService: UsersService,
-    private connectionsService: ConnectionClientService,
-    private connectorService: ConnectorClientService,
-  ) {}
-  async synchronizeConnections() {
-    const { connectors } = await this.connectorService
-      .getAllConnectors({
-        search: undefined,
-        filters: undefined,
-        language: 'pt-br',
-        page: 1,
-        size: 500,
-      })
-      .then(({ connectors }) => ({ connectors: JSON.parse(connectors) }));
-    const { customers } = await this.usersService.findaAllCustomers();
-    const promises: Promise<{
-      customer_name: string;
-      productConnections: Record<string, any>;
-      platformConnections: Record<string, any>;
-    }>[] = [];
-    customers.forEach((c) => {
-      const customer_name = c.name;
-      promises.push(
-        this.connectionsService
-          .INTERNAL_getAllConnections({ customer_name })
-          .then((res) => ({
-            customer_name,
-            platformConnections: JSON.parse(res.platformConnections),
-            productConnections: JSON.parse(res.productConnections),
-          })),
-      );
-    });
-    const customer_connections = await Promise.all(promises);
-    return { customer_connections, connectors };
-  }
-}

src/modules/connection/client.service.ts

@@ -8,8 +8,10 @@ import { ClientGrpc } from '@nestjs/microservices';
 import { ConnectionManager } from '@dadosfera/protospack-v2';
 import { lastValueFrom } from 'rxjs';
 import {
+  ConnectionApiConnection,
   ConnectionRes,
   ConnectionsRes,
+  ConnectionToCatalogDto,
   CreateConnectionDto,
   UpdateConnectionDto,
 } from './dtos/connection';
@@ -18,6 +20,12 @@ import {
   GetConnectionDetailsResponse,
   INTERNAL_GetAllConnectionsResponse,
 } from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
+import { UsersService } from '../users/users.service';
+import { ConnectorClientService } from '../connector/client.service';
+import {
+  ConnectionsApiNetworkConfig,
+  NetworkConfigDto,
+} from '../network-config/dto/network-config';
 
 export class ConnectionClientService implements OnModuleInit {
   private connectionServiceRead: ConnectionManager.ReadService.ConnectionManagerReadServices;
@@ -25,6 +33,8 @@ export class ConnectionClientService implements OnModuleInit {
   constructor(
     @Inject(ConnectionClientConfiguration.name)
     private readonly grpcClient: ClientGrpc,
+    private usersService: UsersService,
+    private connectorService: ConnectorClientService,
   ) {}
 
   onModuleInit() {
@@ -163,4 +173,160 @@ export class ConnectionClientService implements OnModuleInit {
       this.connectionServiceRead.INTERNAL_GetAllConnections({ customer_name }),
     );
   }
+
+  async catalogExistingConnection(data: {
+    connection?: ConnectionToCatalogDto;
+    networkConfig?: NetworkConfigDto;
+  }) {
+    return lastValueFrom(
+      this.connectionServiceRead.INTERNAL_CatalogExistingConnection({
+        data_str: JSON.stringify(data),
+      }),
+    );
+  }
+
+  async synchronizeConnections(apply: boolean) {
+    const connectors = await this.connectorService
+      .getAllConnectors({
+        search: undefined,
+        filters: undefined,
+        language: 'pt-br',
+        page: 1,
+        size: 500,
+      })
+      .then(({ connectors: connectorsStr }) =>
+        JSON.parse(connectorsStr).map((c) => c),
+      );
+    const { customers } = await this.usersService.findAllCustomers();
+    const promises: Promise<{
+      customer_name: string;
+      customer_id: string;
+      productConnections: Record<string, any>[];
+      platformConnections: ConnectionApiConnection[];
+      productNetworkConfigs: Record<string, any>[];
+      platformNetworkConfigs: ConnectionsApiNetworkConfig[];
+    }>[] = [];
+
+    customers.forEach((c) => {
+      const customer_name = c.name;
+      const customer_id = c.id;
+      promises.push(
+        this.INTERNAL_getAllConnections({ customer_name }).then((res) => ({
+          customer_name,
+          customer_id,
+          platformConnections: JSON.parse(res.platformConnections),
+          productConnections: JSON.parse(res.productConnections),
+          productNetworkConfigs: JSON.parse(res.productNetworkConfigs),
+          platformNetworkConfigs: JSON.parse(res.platformNetworkConfigs),
+        })),
+      );
+    });
+    const customer_connections = await Promise.all(promises);
+
+    const productConnectionsMissing: ConnectionApiConnection[] = [];
+    const productNetworkConfigsMissing: ConnectionsApiNetworkConfig[] = [];
+    const migrations = customer_connections.map((cc) => {
+      const {
+        customer_name,
+        productConnections,
+        platformConnections,
+        productNetworkConfigs,
+        platformNetworkConfigs,
+        customer_id,
+      } = cc;
+
+      platformConnections.forEach((plc) => {
+        const productConnection = productConnections.find(
+          (prc) => prc.id === plc.config_id,
+        );
+        if (!productConnection) productConnectionsMissing.push(plc);
+      });
+
+      platformNetworkConfigs.forEach((platNetConfig) => {
+        const productNetworkConfig = productNetworkConfigs.find(
+          (prodNetConfig) => prodNetConfig.id === platNetConfig.config_id,
+        );
+        if (!productNetworkConfig)
+          productNetworkConfigsMissing.push(platNetConfig);
+      });
+
+      const updatePromises: Promise<any>[] = [];
+      for (const network_config of productNetworkConfigsMissing) {
+        const networkConfigToCatalog = {
+          id: network_config.config_id,
+          created_at: network_config.created_at,
+          updated_at: network_config.updated_at,
+          customer_name: cc.customer_name,
+          customer_id: cc.customer_id,
+          user_id: '',
+          network_type_id: '',
+          network_type_name: network_config.type,
+          type: network_config.type,
+          plugin: network_config.plugin,
+          name: network_config.name,
+          description: network_config.description,
+          properties: undefined,
+          keywords: undefined,
+          tags: undefined,
+        };
+
+        if (apply) {
+          updatePromises.push(
+            this.catalogExistingConnection({
+              networkConfig: networkConfigToCatalog,
+            }),
+          );
+        }
+      }
+      for (const connection of productConnectionsMissing) {
+        const connector = connectors.find(
+          (c) => c._source.plugin === connection.plugin,
+        );
+        const connectionToCatalog = {
+          id: connection.config_id,
+          created_at: connection.created_at,
+          updated_at: new Date().toISOString(),
+          customer_id: customer_id,
+          user_id: '',
+          plugin: connection.plugin,
+          name: connection.name,
+          description: connection.description,
+          image_url: connector._source.image,
+          network_config_id: connection.network_config_id,
+          type: connection.type,
+          connector_id: connector._id,
+          connector_name: connector._source.name,
+          in_use: 1,
+          username: 'auto',
+          customer_name: customer_name,
+        };
+
+        if (apply) {
+          updatePromises.push(
+            this.catalogExistingConnection({ connection: connectionToCatalog }),
+          );
+        }
+      }
+      return {
+        customer: { ...cc },
+        updatePromises,
+        productConnectionsMissing,
+        productNetworkConfigsMissing,
+      };
+    });
+    const customersStatus: any[] = [];
+    for (const migration of migrations) {
+      let status: any[] = [];
+      if (apply) {
+        status = await Promise.all(migration.updatePromises);
+      }
+      customersStatus.push({
+        customer: migration.customer.customer_name,
+        status,
+        productConnectionsMissing: migration.productConnectionsMissing,
+        productNetworkConfigsMissing: migration.productNetworkConfigsMissing,
+      });
+    }
+    return { customersStatus };
+  }
 }

src/modules/connection/connection.controller.ts

@@ -44,6 +44,12 @@ export class ConnectionController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @Get('connections-migration')
+  @RequireAllPermissions(PERMISSIONS_GROUPS.DADOSFERA.permissions.ADMIN)
+  async synchronizeConnections(@Query('apply') apply: string) {
+    return this.clientService.synchronizeConnections(apply === 'true');
+  }
+
   @Post()
   @RequireAllPermissions(connectionPermissions.CREATE)
   async createConnection(

src/modules/connection/connection.module.ts

@@ -4,13 +4,19 @@ import { ConnectionClientService } from './client.service';
 import { ConnectionClientConfiguration } from './client.config';
 import { ConnectionController } from './connection.controller';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
+import { UsersModule } from '../users/users.module';
+import { ConnectorModule } from '../connector/connector.module';
 
 const client = new ConnectionClientConfiguration();
 
 @Module({
+  imports: [
+    ClientsModule.register([client.providerOptions]),
+    UsersModule,
+    ConnectorModule,
+  ],
   controllers: [ConnectionController],
   providers: [ConnectionClientService, DadosferaLogger],
-  imports: [ClientsModule.register([client.providerOptions])],
   exports: [ConnectionClientService],
 })
 export class ConnectionModule {}

src/modules/connection/dtos/connection.ts

@@ -13,7 +13,30 @@ import {
   GetConnectionDetailsResponse,
   GetConnectionResponse,
 } from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/messages';
-
+export type ConnectionTypes = 'database' | 'file' | 'application';
+export type ConnectionCredentialsType =
+  | 'basic_auth'
+  | 'iam_user'
+  | 'role_name'
+  | 'oauth'
+  | 'api_key'
+  | 'service_account'
+  | 'headers_authF';
+export interface ConnectionApiConnection {
+  config_id: string;
+  plugin: string;
+  secret_id: string;
+  created_at: string;
+  network_config_id: null;
+  type: ConnectionTypes;
+  credentials_type: ConnectionCredentialsType;
+  credentials: Record<string, any>;
+  sensitive_fields: string[];
+  name: string;
+  customer_name: string;
+  description: string;
+  updated_at: string;
+}
 export class ConnectionDto implements Connection {
   // ---Automatically generated information will not be sent by the frontend--- //
   @ApiProperty()
@@ -70,21 +93,17 @@ export class ConnectionDto implements Connection {
 }
 
 export class ConnectionToCatalogDto implements ConnectionToCatalog {
-  // ---Automatically generated information will not be sent by the frontend--- //
-  @ApiProperty()
+  // ---Automatically generated information - will not be sent by the frontend--- //
   id: string;
-
-  @ApiProperty()
   customer_id: string;
-
-  @ApiProperty()
   updated_at: string;
-
-  @ApiProperty()
   created_at: string;
+  // ---Information extracted from token--- //
+  user_id: string;
+  username: string;
+  customer_name: string;
 
   // ---Information sent by the frontend--- //
-
   @ApiProperty()
   name: string;
 
@@ -98,7 +117,6 @@ export class ConnectionToCatalogDto implements ConnectionToCatalog {
   network_config_id: string | null;
 
   // ---Information sent by the frontend (Heirs of Connector)--- //
-
   @ApiProperty()
   connector_id: string;
 

src/modules/network-config/dto/network-config.ts

@@ -2,7 +2,12 @@ import {
   NetworkConfigEntity,
   NetworkConfigToCatalog,
 } from '@dadosfera/protospack-v2/dist/lib/NetworkConfig/interfaces/entities';
-import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
+import {
+  ApiProperty,
+  ApiPropertyOptional,
+  OmitType,
+  PickType,
+} from '@nestjs/swagger';
 
 export enum NetworkConfigTypesEnum {
   vpn = 'vpn',
@@ -10,6 +15,24 @@ export enum NetworkConfigTypesEnum {
 export enum NetworkConfigPluginsEnum {
   openvpn = 'openvpn',
 }
+export interface ConnectionsApiNetworkConfig {
+  config_id: string;
+  created_at: string;
+  credentials: {
+    CERTIFICATE_FILE: string;
+    OVPN_CREDENTIALS: string;
+    TLS_AUTH_FILE: string;
+    OVPN_FILE: string;
+  };
+  customer_name: string;
+  description: string;
+  name: string;
+  plugin: string;
+  secret_id: string;
+  sensitive_fields: string[];
+  type: string;
+  updated_at: string;
+}
 export class NetworkConfigDto implements NetworkConfigEntity {
   @ApiProperty()
   id: string;

src/modules/users/users.service.ts

@@ -208,7 +208,7 @@ export class UsersService implements OnModuleInit {
     });
   }
 
-  async findaAllCustomers() {
+  async findAllCustomers() {
     return await lastValueFrom(this.customersClientService.CustomerFindAll({}));
   }
 }