mirror of
https://github.com/dadosfera/maestro.git
synced 2026-10-31 05:49:08 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6a90238908 | ||
|
|
fd1dfdd9d8 | ||
|
|
2e88843980 | ||
|
|
1c3ee79827 | ||
|
|
d35beb1f08 |
@@ -111,6 +111,8 @@ spec:
|
|||||||
value: "{{ .Values.maestro.redis_tls }}"
|
value: "{{ .Values.maestro.redis_tls }}"
|
||||||
- name: PLATFORM_API_URL
|
- name: PLATFORM_API_URL
|
||||||
value: {{ .Values.maestro.platform_api_url }}
|
value: {{ .Values.maestro.platform_api_url }}
|
||||||
|
- name: CONNECTIONS_API_URL
|
||||||
|
value: {{ .Values.maestro.connections_api_url | default "" | quote }}
|
||||||
- name: STORAGE_EXPLORER_API_URL
|
- name: STORAGE_EXPLORER_API_URL
|
||||||
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
|
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
|
||||||
- name: FIREBASE_BASE_URL
|
- name: FIREBASE_BASE_URL
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ maestro:
|
|||||||
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
|
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
|
||||||
redis_database: "1"
|
redis_database: "1"
|
||||||
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
|
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
|
||||||
|
connections_api_url: https://iy40eans64.execute-api.us-east-1.amazonaws.com
|
||||||
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
|
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
|
||||||
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
|
firebase_base_url: https://feature-flag-25bf6-default-rtdb.firebaseio.com/stg
|
||||||
|
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ maestro:
|
|||||||
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
|
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
|
||||||
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
||||||
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
|
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
|
||||||
|
connections_api_url: https://wv6fw7ayj4.execute-api.us-east-1.amazonaws.com
|
||||||
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
|
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
|
||||||
dedicated_proxy: ""
|
dedicated_proxy: ""
|
||||||
restricted_ip: ""
|
restricted_ip: ""
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ import {
|
|||||||
} from '@nestjs/microservices';
|
} from '@nestjs/microservices';
|
||||||
import { ConnectionTest } from '@dadosfera/protospack-v2';
|
import { ConnectionTest } from '@dadosfera/protospack-v2';
|
||||||
|
|
||||||
|
const infactoryUrl = process.env.INFACTORY_URL ?? '';
|
||||||
const isLocalConnection =
|
const isLocalConnection =
|
||||||
process.env.INFACTORY_URL.startsWith('in-factory:') ||
|
infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
|
||||||
process.env.INFACTORY_URL.includes('0.0.0.0');
|
|
||||||
|
|
||||||
export class ConnectionTestClientConfiguration {
|
export class ConnectionTestClientConfiguration {
|
||||||
private config: GrpcOptions = {
|
private config: GrpcOptions = {
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ import {
|
|||||||
ConnectionTestListTablesRes,
|
ConnectionTestListTablesRes,
|
||||||
GetTableMetadataRes,
|
GetTableMetadataRes,
|
||||||
GetTableMetadataReq,
|
GetTableMetadataReq,
|
||||||
|
RefreshCatalogReq,
|
||||||
|
RefreshCatalogRes,
|
||||||
|
RefreshCatalogStatusReq,
|
||||||
} from './dto/connection-test';
|
} from './dto/connection-test';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||||
@@ -84,7 +87,7 @@ export class ConnectionTestController {
|
|||||||
});
|
});
|
||||||
return this.connectionTestService.connectionTestListSchemas(
|
return this.connectionTestService.connectionTestListSchemas(
|
||||||
body,
|
body,
|
||||||
user.customer_name,
|
user,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,7 +104,7 @@ export class ConnectionTestController {
|
|||||||
});
|
});
|
||||||
return this.connectionTestService.connectionTestListTables(
|
return this.connectionTestService.connectionTestListTables(
|
||||||
body,
|
body,
|
||||||
user.customer_name,
|
user,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,7 +121,38 @@ export class ConnectionTestController {
|
|||||||
});
|
});
|
||||||
return this.connectionTestService.getTableMetadata(
|
return this.connectionTestService.getTableMetadata(
|
||||||
body,
|
body,
|
||||||
user.customer_name,
|
user,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Post('refresh-catalog')
|
||||||
|
@ApiOkResponse({ type: RefreshCatalogRes })
|
||||||
|
@HttpCode(HttpStatus.ACCEPTED)
|
||||||
|
async refreshCatalog(
|
||||||
|
@User() user: RequestUser,
|
||||||
|
@Body(new ValidationPipe()) body: RefreshCatalogReq,
|
||||||
|
) {
|
||||||
|
this.logger.info('/connection-test/refresh-catalog', {
|
||||||
|
user: user.user_id,
|
||||||
|
customer: user.customer_name,
|
||||||
|
connection: body.connection_id,
|
||||||
|
});
|
||||||
|
return this.connectionTestService.refreshCatalog(body, user);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Post('refresh-catalog/status')
|
||||||
|
@ApiOkResponse({ type: RefreshCatalogRes })
|
||||||
|
@HttpCode(HttpStatus.OK)
|
||||||
|
async refreshCatalogStatus(
|
||||||
|
@User() user: RequestUser,
|
||||||
|
@Body(new ValidationPipe()) body: RefreshCatalogStatusReq,
|
||||||
|
) {
|
||||||
|
this.logger.info('/connection-test/refresh-catalog/status', {
|
||||||
|
user: user.user_id,
|
||||||
|
customer: user.customer_name,
|
||||||
|
connection: body.connection_id,
|
||||||
|
session: body.session_id,
|
||||||
|
});
|
||||||
|
return this.connectionTestService.refreshCatalogStatus(body, user);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,10 +5,17 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|||||||
import { ClientsModule } from '@nestjs/microservices';
|
import { ClientsModule } from '@nestjs/microservices';
|
||||||
import { ConnectionTestClientConfiguration } from './connection-test-client.config';
|
import { ConnectionTestClientConfiguration } from './connection-test-client.config';
|
||||||
import { ConnectionModule } from '../connection/connection.module';
|
import { ConnectionModule } from '../connection/connection.module';
|
||||||
|
import { ConnectionsApiModule } from '../connections-api/connections-api.module';
|
||||||
|
import { PlatformApiModule } from '../platform-api/platform-api.module';
|
||||||
const client = new ConnectionTestClientConfiguration();
|
const client = new ConnectionTestClientConfiguration();
|
||||||
@Module({
|
@Module({
|
||||||
controllers: [ConnectionTestController],
|
controllers: [ConnectionTestController],
|
||||||
providers: [ConnectionTestService, DadosferaLogger],
|
providers: [ConnectionTestService, DadosferaLogger],
|
||||||
imports: [ClientsModule.register([client.providerOptions]), ConnectionModule],
|
imports: [
|
||||||
|
ClientsModule.register([client.providerOptions]),
|
||||||
|
ConnectionModule,
|
||||||
|
ConnectionsApiModule,
|
||||||
|
PlatformApiModule,
|
||||||
|
],
|
||||||
})
|
})
|
||||||
export class ConnectionTestModule {}
|
export class ConnectionTestModule {}
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
import { ConnectionTestService } from './connection-test.service';
|
||||||
|
import { RequestUser } from 'src/decorators/user.decorator';
|
||||||
|
|
||||||
|
describe('ConnectionTestService catalog cache', () => {
|
||||||
|
const user: RequestUser = {
|
||||||
|
user_id: 'user-id',
|
||||||
|
username: 'user@example.com',
|
||||||
|
permissions: [],
|
||||||
|
customer_id: 'customer-id',
|
||||||
|
customer_name: 'customer-name',
|
||||||
|
customer_tier: 'standard',
|
||||||
|
access_token: 'token',
|
||||||
|
customer_modules: [],
|
||||||
|
roles: [],
|
||||||
|
};
|
||||||
|
const grpcClient = { getService: jest.fn().mockReturnValue({}) };
|
||||||
|
const connectionsService = {};
|
||||||
|
const connectionsApiService = { proxy: jest.fn() };
|
||||||
|
const platformApiService = { proxy: jest.fn() };
|
||||||
|
let service: ConnectionTestService;
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
jest.clearAllMocks();
|
||||||
|
service = new ConnectionTestService(
|
||||||
|
grpcClient as any,
|
||||||
|
connectionsService as any,
|
||||||
|
connectionsApiService as any,
|
||||||
|
platformApiService as any,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps the existing schemas response contract', async () => {
|
||||||
|
connectionsApiService.proxy.mockResolvedValue({
|
||||||
|
schemas: [{ schema_name: 'analytics' }, { schema_name: 'public' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.connectionTestListSchemas(
|
||||||
|
{ connection_id: 'config-id', plugin: 'postgresql' },
|
||||||
|
user,
|
||||||
|
),
|
||||||
|
).resolves.toEqual({
|
||||||
|
operation_result: true,
|
||||||
|
schema_list: ['analytics', 'public'],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps the existing tables response contract', async () => {
|
||||||
|
connectionsApiService.proxy.mockResolvedValue({
|
||||||
|
tables: [{ table_name: 'customers' }, { table_name: 'orders' }],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.connectionTestListTables(
|
||||||
|
{
|
||||||
|
connection_id: 'config-id',
|
||||||
|
plugin: 'postgresql',
|
||||||
|
schema: 'public',
|
||||||
|
},
|
||||||
|
user,
|
||||||
|
),
|
||||||
|
).resolves.toEqual({
|
||||||
|
operation_result: true,
|
||||||
|
table_list: ['customers', 'orders'],
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('maps cached columns to the existing table metadata contract', async () => {
|
||||||
|
connectionsApiService.proxy.mockResolvedValue({
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
column_name: 'id',
|
||||||
|
data_type: 'bigint',
|
||||||
|
is_primary_key: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.getTableMetadata(
|
||||||
|
{
|
||||||
|
connection_id: 'config-id',
|
||||||
|
plugin: 'postgresql',
|
||||||
|
schema: 'public',
|
||||||
|
table_list: ['customers'],
|
||||||
|
},
|
||||||
|
user,
|
||||||
|
),
|
||||||
|
).resolves.toEqual({
|
||||||
|
operation_result: true,
|
||||||
|
tables_metadata: [
|
||||||
|
{
|
||||||
|
table_name: 'customers',
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
name: 'id',
|
||||||
|
type: 'bigint',
|
||||||
|
is_primary_key: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
references: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
|
||||||
|
'GET',
|
||||||
|
'/connection_catalog/config-id/schemas/public/tables/customers/columns',
|
||||||
|
user,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('submits a catalog refresh without holding the request open', async () => {
|
||||||
|
platformApiService.proxy.mockResolvedValue({
|
||||||
|
session_id: 'session-id',
|
||||||
|
date: '20260731',
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.refreshCatalog(
|
||||||
|
{ connection_id: 'config-id', plugin: 'postgresql' },
|
||||||
|
user,
|
||||||
|
),
|
||||||
|
).resolves.toEqual({
|
||||||
|
operation_result: true,
|
||||||
|
status: 'PENDING',
|
||||||
|
session_id: 'session-id',
|
||||||
|
date: '20260731',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(platformApiService.proxy).toHaveBeenCalledWith(
|
||||||
|
'POST',
|
||||||
|
'/connection_test',
|
||||||
|
user,
|
||||||
|
{
|
||||||
|
customer_id: user.customer_name,
|
||||||
|
plugin: 'postgresql',
|
||||||
|
task: {
|
||||||
|
task_type: 'refresh_catalog',
|
||||||
|
connection: {
|
||||||
|
provider: 'connection_manager',
|
||||||
|
config_id: 'config-id',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps polling without changing the catalog pointer while pending', async () => {
|
||||||
|
platformApiService.proxy.mockResolvedValue({ status: 'PENDING' });
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.refreshCatalogStatus(
|
||||||
|
{
|
||||||
|
connection_id: 'config-id',
|
||||||
|
plugin: 'postgresql',
|
||||||
|
session_id: 'session-id',
|
||||||
|
date: '20260731',
|
||||||
|
},
|
||||||
|
user,
|
||||||
|
),
|
||||||
|
).resolves.toEqual({
|
||||||
|
operation_result: false,
|
||||||
|
status: 'PENDING',
|
||||||
|
session_id: 'session-id',
|
||||||
|
date: '20260731',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(connectionsApiService.proxy).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('publishes the catalog pointer after the refresh finishes', async () => {
|
||||||
|
platformApiService.proxy.mockResolvedValue({ status: 'DONE' });
|
||||||
|
connectionsApiService.proxy.mockResolvedValue({
|
||||||
|
last_catalog_refresh_status: 'SUCCESS',
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.refreshCatalogStatus(
|
||||||
|
{
|
||||||
|
connection_id: 'config/id',
|
||||||
|
plugin: 'postgresql',
|
||||||
|
session_id: 'session-id',
|
||||||
|
date: '20260731',
|
||||||
|
},
|
||||||
|
user,
|
||||||
|
),
|
||||||
|
).resolves.toEqual({
|
||||||
|
operation_result: true,
|
||||||
|
status: 'DONE',
|
||||||
|
session_id: 'session-id',
|
||||||
|
date: '20260731',
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(connectionsApiService.proxy).toHaveBeenCalledWith(
|
||||||
|
'PUT',
|
||||||
|
'/connection_config/config%2Fid/catalog_metadata',
|
||||||
|
user,
|
||||||
|
{
|
||||||
|
last_catalog_refresh_status: 'SUCCESS',
|
||||||
|
last_catalog_connection_test_date: '20260731',
|
||||||
|
last_catalog_connection_test_session_id: 'session-id',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common';
|
||||||
import { ClientGrpc } from '@nestjs/microservices';
|
import { ClientGrpc } from '@nestjs/microservices';
|
||||||
import { ConnectionTest } from '@dadosfera/protospack-v2';
|
import { ConnectionTest } from '@dadosfera/protospack-v2';
|
||||||
import { lastValueFrom } from 'rxjs';
|
import { lastValueFrom } from 'rxjs';
|
||||||
@@ -13,6 +13,9 @@ import {
|
|||||||
ConnectionTestPingRes,
|
ConnectionTestPingRes,
|
||||||
GetTableMetadataReq,
|
GetTableMetadataReq,
|
||||||
GetTableMetadataRes,
|
GetTableMetadataRes,
|
||||||
|
RefreshCatalogReq,
|
||||||
|
RefreshCatalogRes,
|
||||||
|
RefreshCatalogStatusReq,
|
||||||
} from './dto/connection-test';
|
} from './dto/connection-test';
|
||||||
import { ConnectionClientService } from '../connection/client.service';
|
import { ConnectionClientService } from '../connection/client.service';
|
||||||
import {
|
import {
|
||||||
@@ -21,6 +24,8 @@ import {
|
|||||||
} from '../connection/dtos/connection';
|
} from '../connection/dtos/connection';
|
||||||
import { RequestUser } from 'src/decorators/user.decorator';
|
import { RequestUser } from 'src/decorators/user.decorator';
|
||||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||||
|
import { ConnectionsApiService } from '../connections-api/connections-api.service';
|
||||||
|
import { PlatformApiService } from '../platform-api/platform-api.service';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ConnectionTestService {
|
export class ConnectionTestService {
|
||||||
@@ -28,6 +33,8 @@ export class ConnectionTestService {
|
|||||||
constructor(
|
constructor(
|
||||||
@Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
|
@Inject('ConnectionTestGrpcClient') private readonly grpcClient: ClientGrpc,
|
||||||
private connectionsService: ConnectionClientService,
|
private connectionsService: ConnectionClientService,
|
||||||
|
private connectionsApiService: ConnectionsApiService,
|
||||||
|
private platformApiService: PlatformApiService,
|
||||||
) {
|
) {
|
||||||
this.connectionTestReadClient =
|
this.connectionTestReadClient =
|
||||||
grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
|
grpcClient.getService<ConnectionTest.ReadService.ConnectionTestReadServices>(
|
||||||
@@ -147,45 +154,137 @@ export class ConnectionTestService {
|
|||||||
}
|
}
|
||||||
async connectionTestListSchemas(
|
async connectionTestListSchemas(
|
||||||
body: ConnectionTestListSchemasReq,
|
body: ConnectionTestListSchemasReq,
|
||||||
customer_name: string,
|
user: RequestUser,
|
||||||
): Promise<ConnectionTestListSchemasRes> {
|
): Promise<ConnectionTestListSchemasRes> {
|
||||||
const { connection_id, plugin } = body;
|
const result = await this.connectionsApiService.proxy(
|
||||||
return lastValueFrom(
|
'GET',
|
||||||
this.connectionTestReadClient.ListSchemas({
|
`/connection_catalog/${encodeURIComponent(body.connection_id)}/schemas`,
|
||||||
connection_id,
|
user,
|
||||||
customer_name,
|
|
||||||
plugin,
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
|
return {
|
||||||
|
operation_result: true,
|
||||||
|
schema_list: result.schemas.map((schema) => schema.schema_name),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async connectionTestListTables(
|
async connectionTestListTables(
|
||||||
body: ConnectionTestListTablesReq,
|
body: ConnectionTestListTablesReq,
|
||||||
customer_name: string,
|
user: RequestUser,
|
||||||
): Promise<ConnectionTestListTablesRes> {
|
): Promise<ConnectionTestListTablesRes> {
|
||||||
const { connection_id, plugin, schema } = body;
|
const result = await this.connectionsApiService.proxy(
|
||||||
return lastValueFrom(
|
'GET',
|
||||||
this.connectionTestReadClient.ListTables({
|
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
|
||||||
connection_id,
|
`/schemas/${encodeURIComponent(body.schema)}/tables`,
|
||||||
customer_name,
|
user,
|
||||||
plugin,
|
|
||||||
schema,
|
|
||||||
}),
|
|
||||||
);
|
);
|
||||||
|
return {
|
||||||
|
operation_result: true,
|
||||||
|
table_list: result.tables.map((table) => table.table_name),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async getTableMetadata(
|
async getTableMetadata(
|
||||||
body: GetTableMetadataReq,
|
body: GetTableMetadataReq,
|
||||||
customer_name: string,
|
user: RequestUser,
|
||||||
): Promise<GetTableMetadataRes> {
|
): Promise<GetTableMetadataRes> {
|
||||||
const { schema, plugin, table_list, connection_id } = body;
|
const tables_metadata = await Promise.all(
|
||||||
return lastValueFrom(
|
body.table_list.map(async (table_name) => {
|
||||||
this.connectionTestReadClient.GetTableMetadata({
|
const result = await this.connectionsApiService.proxy(
|
||||||
connection_id,
|
'GET',
|
||||||
customer_name,
|
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
|
||||||
plugin,
|
`/schemas/${encodeURIComponent(body.schema)}` +
|
||||||
schema,
|
`/tables/${encodeURIComponent(table_name)}/columns`,
|
||||||
table_list,
|
user,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
table_name,
|
||||||
|
columns: result.columns.map((column) => ({
|
||||||
|
name: column.column_name,
|
||||||
|
type: column.data_type,
|
||||||
|
is_primary_key: column.is_primary_key,
|
||||||
|
})),
|
||||||
|
references: [],
|
||||||
|
};
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
return { operation_result: true, tables_metadata };
|
||||||
|
}
|
||||||
|
|
||||||
|
async refreshCatalog(
|
||||||
|
body: RefreshCatalogReq,
|
||||||
|
user: RequestUser,
|
||||||
|
): Promise<RefreshCatalogRes> {
|
||||||
|
const task = await this.platformApiService.proxy(
|
||||||
|
'POST',
|
||||||
|
'/connection_test',
|
||||||
|
user,
|
||||||
|
{
|
||||||
|
customer_id: user.customer_name,
|
||||||
|
plugin: body.plugin,
|
||||||
|
task: {
|
||||||
|
task_type: 'refresh_catalog',
|
||||||
|
connection: {
|
||||||
|
provider: 'connection_manager',
|
||||||
|
config_id: body.connection_id,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!task.session_id || !task.date) {
|
||||||
|
throw new HttpException(
|
||||||
|
'Platform API did not return a catalog refresh task identifier',
|
||||||
|
HttpStatus.BAD_GATEWAY,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
operation_result: true,
|
||||||
|
status: 'PENDING',
|
||||||
|
session_id: task.session_id,
|
||||||
|
date: task.date,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async refreshCatalogStatus(
|
||||||
|
body: RefreshCatalogStatusReq,
|
||||||
|
user: RequestUser,
|
||||||
|
): Promise<RefreshCatalogRes> {
|
||||||
|
const result = await this.platformApiService.proxy(
|
||||||
|
'POST',
|
||||||
|
'/connection_test/status',
|
||||||
|
user,
|
||||||
|
{
|
||||||
|
session_id: body.session_id,
|
||||||
|
date: body.date,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result.status === 'DONE') {
|
||||||
|
await this.connectionsApiService.proxy(
|
||||||
|
'PUT',
|
||||||
|
`/connection_config/${encodeURIComponent(
|
||||||
|
body.connection_id,
|
||||||
|
)}/catalog_metadata`,
|
||||||
|
user,
|
||||||
|
{
|
||||||
|
last_catalog_refresh_status: 'SUCCESS',
|
||||||
|
last_catalog_connection_test_date: body.date,
|
||||||
|
last_catalog_connection_test_session_id: body.session_id,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} else if (result.status === 'ERROR' || result.status === 'EXPIRED') {
|
||||||
|
throw new HttpException(
|
||||||
|
`Catalog refresh finished with status ${result.status}`,
|
||||||
|
HttpStatus.BAD_GATEWAY,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
operation_result: result.status === 'DONE',
|
||||||
|
status: result.status,
|
||||||
|
session_id: body.session_id,
|
||||||
|
date: body.date,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||||
import { IsString, IsOptional } from 'class-validator';
|
import { IsIn, IsString, IsOptional } from 'class-validator';
|
||||||
import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
|
import { DatabaseConnectionPropertiesDto } from 'src/modules/connection/dtos/connection';
|
||||||
import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
|
import { CreateConnectionDto } from 'src/modules/connection/dtos/connection';
|
||||||
export class ColumnDto {
|
export class ColumnDto {
|
||||||
@@ -7,6 +7,8 @@ export class ColumnDto {
|
|||||||
name: string;
|
name: string;
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
type: string;
|
type: string;
|
||||||
|
@ApiProperty()
|
||||||
|
is_primary_key: boolean;
|
||||||
}
|
}
|
||||||
export class TableMetadataDto {
|
export class TableMetadataDto {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
@@ -131,3 +133,37 @@ export class GetTableMetadataRes {
|
|||||||
@ApiProperty({ type: [TableMetadataDto] })
|
@ApiProperty({ type: [TableMetadataDto] })
|
||||||
tables_metadata: TableMetadataDto[];
|
tables_metadata: TableMetadataDto[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class RefreshCatalogReq {
|
||||||
|
@ApiProperty()
|
||||||
|
@IsString()
|
||||||
|
connection_id: string;
|
||||||
|
|
||||||
|
@ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] })
|
||||||
|
@IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver'])
|
||||||
|
plugin: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RefreshCatalogStatusReq extends RefreshCatalogReq {
|
||||||
|
@ApiProperty()
|
||||||
|
@IsString()
|
||||||
|
session_id: string;
|
||||||
|
|
||||||
|
@ApiProperty()
|
||||||
|
@IsString()
|
||||||
|
date: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class RefreshCatalogRes {
|
||||||
|
@ApiProperty()
|
||||||
|
operation_result: boolean;
|
||||||
|
|
||||||
|
@ApiProperty()
|
||||||
|
status: string;
|
||||||
|
|
||||||
|
@ApiProperty()
|
||||||
|
session_id: string;
|
||||||
|
|
||||||
|
@ApiProperty()
|
||||||
|
date: string;
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ import {
|
|||||||
} from '@nestjs/microservices';
|
} from '@nestjs/microservices';
|
||||||
import { ConnectionManager } from '@dadosfera/protospack-v2';
|
import { ConnectionManager } from '@dadosfera/protospack-v2';
|
||||||
|
|
||||||
|
const infactoryUrl = process.env.INFACTORY_URL ?? '';
|
||||||
const isLocalConnection =
|
const isLocalConnection =
|
||||||
process.env.INFACTORY_URL.startsWith('in-factory:') ||
|
infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
|
||||||
process.env.INFACTORY_URL.includes('0.0.0.0');
|
|
||||||
|
|
||||||
export class ConnectionClientConfiguration {
|
export class ConnectionClientConfiguration {
|
||||||
public name = 'ConnectionClientConfiguration';
|
public name = 'ConnectionClientConfiguration';
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
export const CONNECTIONS_API_CONFIG = {
|
||||||
|
getUrl: (): string => {
|
||||||
|
const url = process.env.CONNECTIONS_API_URL;
|
||||||
|
if (!url) {
|
||||||
|
throw new Error('CONNECTIONS_API_URL environment variable is not set');
|
||||||
|
}
|
||||||
|
return url;
|
||||||
|
},
|
||||||
|
region: process.env.AWS_REGION || 'us-east-1',
|
||||||
|
timeout: parseInt(process.env.CONNECTIONS_API_TIMEOUT || '30000', 10),
|
||||||
|
};
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
|
import { ConnectionsApiService } from './connections-api.service';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
providers: [ConnectionsApiService, DadosferaLogger],
|
||||||
|
exports: [ConnectionsApiService],
|
||||||
|
})
|
||||||
|
export class ConnectionsApiModule {}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||||
|
import { SignatureV4 } from '@aws-sdk/signature-v4';
|
||||||
|
import { Sha256 } from '@aws-crypto/sha256-js';
|
||||||
|
import { defaultProvider } from '@aws-sdk/credential-provider-node';
|
||||||
|
import axios, { AxiosResponse, Method } from 'axios';
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
|
import { RequestUser } from '../../decorators/user.decorator';
|
||||||
|
import { CONNECTIONS_API_CONFIG } from './connections-api.config';
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class ConnectionsApiService {
|
||||||
|
private signer: SignatureV4;
|
||||||
|
private logger: any;
|
||||||
|
|
||||||
|
constructor(@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger) {
|
||||||
|
this.logger = dadosferaLogger.logger;
|
||||||
|
this.signer = new SignatureV4({
|
||||||
|
service: 'execute-api',
|
||||||
|
region: CONNECTIONS_API_CONFIG.region,
|
||||||
|
credentials: defaultProvider(),
|
||||||
|
sha256: Sha256,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async proxy(
|
||||||
|
method: string,
|
||||||
|
path: string,
|
||||||
|
user: RequestUser,
|
||||||
|
body?: any,
|
||||||
|
query?: Record<string, string>,
|
||||||
|
): Promise<any> {
|
||||||
|
const baseUrl = CONNECTIONS_API_CONFIG.getUrl();
|
||||||
|
const url = new URL(`${baseUrl}${path}`);
|
||||||
|
|
||||||
|
if (query) {
|
||||||
|
Object.entries(query).forEach(([key, value]) => {
|
||||||
|
if (value !== undefined && value !== null) {
|
||||||
|
url.searchParams.set(key, String(value));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const headers: Record<string, string> = {
|
||||||
|
host: url.hostname,
|
||||||
|
'content-type': 'application/json',
|
||||||
|
customer_name: user.customer_name || '',
|
||||||
|
customer_id: user.customer_id || '',
|
||||||
|
'x-user-id': user.user_id || '',
|
||||||
|
'x-username': user.username || '',
|
||||||
|
'x-customer-tier': user.customer_tier || '',
|
||||||
|
'x-customer-id': user.customer_id || '',
|
||||||
|
};
|
||||||
|
const requestToSign = {
|
||||||
|
method: method.toUpperCase(),
|
||||||
|
protocol: url.protocol,
|
||||||
|
hostname: url.hostname,
|
||||||
|
port: url.port ? parseInt(url.port, 10) : undefined,
|
||||||
|
path: url.pathname + url.search,
|
||||||
|
headers,
|
||||||
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const signedRequest = await this.signer.sign(requestToSign);
|
||||||
|
const response: AxiosResponse = await axios({
|
||||||
|
method: method as Method,
|
||||||
|
url: url.href,
|
||||||
|
headers: signedRequest.headers as Record<string, string>,
|
||||||
|
data: body,
|
||||||
|
timeout: CONNECTIONS_API_CONFIG.timeout,
|
||||||
|
validateStatus: () => true,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status >= 400) {
|
||||||
|
throw new HttpException(response.data, response.status);
|
||||||
|
}
|
||||||
|
return response.data;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Connections API proxy error', {
|
||||||
|
error: error.message,
|
||||||
|
path,
|
||||||
|
method: method.toUpperCase(),
|
||||||
|
});
|
||||||
|
if (error instanceof HttpException) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
if (error.response) {
|
||||||
|
throw new HttpException(error.response.data, error.response.status);
|
||||||
|
}
|
||||||
|
if (error.code === 'ECONNREFUSED') {
|
||||||
|
throw new HttpException('Connections API service unavailable', 503);
|
||||||
|
}
|
||||||
|
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||||
|
throw new HttpException('Connections API request timeout', 504);
|
||||||
|
}
|
||||||
|
throw new HttpException('Internal server error', 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,9 +6,9 @@ import {
|
|||||||
} from '@nestjs/microservices';
|
} from '@nestjs/microservices';
|
||||||
import { ConnectorManager } from '@dadosfera/protospack-v2';
|
import { ConnectorManager } from '@dadosfera/protospack-v2';
|
||||||
|
|
||||||
|
const infactoryUrl = process.env.INFACTORY_URL ?? '';
|
||||||
const isLocalConnection =
|
const isLocalConnection =
|
||||||
process.env.INFACTORY_URL.startsWith('in-factory:') ||
|
infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
|
||||||
process.env.INFACTORY_URL.includes('0.0.0.0');
|
|
||||||
|
|
||||||
export class ConnectorClientConfiguration {
|
export class ConnectorClientConfiguration {
|
||||||
public name = 'ConnectorClientConfiguration';
|
public name = 'ConnectorClientConfiguration';
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ import {
|
|||||||
type GrpcOptions,
|
type GrpcOptions,
|
||||||
} from '@nestjs/microservices';
|
} from '@nestjs/microservices';
|
||||||
|
|
||||||
|
const infactoryUrl = process.env.INFACTORY_URL ?? '';
|
||||||
const isLocalConnection =
|
const isLocalConnection =
|
||||||
process.env.INFACTORY_URL.startsWith('in-factory:') ||
|
infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
|
||||||
process.env.INFACTORY_URL.includes('0.0.0.0');
|
|
||||||
export class InputsGrpcClient {
|
export class InputsGrpcClient {
|
||||||
public readonly name = 'InputsGrpcClient';
|
public readonly name = 'InputsGrpcClient';
|
||||||
private config: GrpcOptions = {
|
private config: GrpcOptions = {
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ import {
|
|||||||
} from '@nestjs/microservices';
|
} from '@nestjs/microservices';
|
||||||
import { NetworkConfig } from '@dadosfera/protospack-v2';
|
import { NetworkConfig } from '@dadosfera/protospack-v2';
|
||||||
|
|
||||||
|
const infactoryUrl = process.env.INFACTORY_URL ?? '';
|
||||||
const isLocalConnection =
|
const isLocalConnection =
|
||||||
process.env.INFACTORY_URL.startsWith('in-factory:') ||
|
infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
|
||||||
process.env.INFACTORY_URL.includes('0.0.0.0');
|
|
||||||
|
|
||||||
export class NetworkConfigGrpcClient {
|
export class NetworkConfigGrpcClient {
|
||||||
public name = 'NetworkConfigGrpcClient';
|
public name = 'NetworkConfigGrpcClient';
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ import {
|
|||||||
import { credentials } from '@grpc/grpc-js';
|
import { credentials } from '@grpc/grpc-js';
|
||||||
import { Transformation } from '@dadosfera/protospack-v2';
|
import { Transformation } from '@dadosfera/protospack-v2';
|
||||||
|
|
||||||
|
const infactoryUrl = process.env.INFACTORY_URL ?? '';
|
||||||
const isLocalConnection =
|
const isLocalConnection =
|
||||||
process.env.INFACTORY_URL.startsWith('in-factory:') ||
|
infactoryUrl.startsWith('in-factory:') || infactoryUrl.includes('0.0.0.0');
|
||||||
process.env.INFACTORY_URL.includes('0.0.0.0');
|
|
||||||
|
|
||||||
export class TransformationsClientConfiguration {
|
export class TransformationsClientConfiguration {
|
||||||
public name = 'TransformationsClientConfiguration';
|
public name = 'TransformationsClientConfiguration';
|
||||||
|
|||||||
Reference in New Issue
Block a user