mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-16 01:09:09 +00:00
FEAT: add Platform API proxy with Elasticsearch and DynamoDB sync
- Add Platform API module to proxy requests to Platform API service - Add Elasticsearch service for pipeline catalog sync (CRUD operations) - Add DynamoDB service for storing pipeline tables/inputs configuration - Sync pipeline creation/update/delete to Elasticsearch index - Extract tables from jobs with connector-specific mappings: - JDBC: table_name, load_type, column_include_list, incremental_column_name - Singer: replication_method -> full_load/incremental - S3: same as Singer - Map connector types to DynamoDB types (jdbc->database, singer->application, s3->file) - Validate connector type is provided in job input - Normalize pipeline IDs for Platform API (replace - with _), keep UUIDs for ES 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.5
parent
e789076ed4
commit
df674dd441
@@ -0,0 +1,4 @@
|
||||
export const DYNAMODB_CONFIG = {
|
||||
region: () => process.env.AWS_REGION || 'us-east-1',
|
||||
inputsTable: () => process.env.INPUTS_DB || 'dadosfera-inputs-prd',
|
||||
};
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { DynamoDBService } from './dynamodb.service';
|
||||
|
||||
@Module({
|
||||
providers: [DynamoDBService, DadosferaLogger],
|
||||
exports: [DynamoDBService],
|
||||
})
|
||||
export class DynamoDBModule {}
|
||||
@@ -0,0 +1,148 @@
|
||||
import { Injectable, Inject } from '@nestjs/common';
|
||||
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
|
||||
import {
|
||||
DynamoDBDocumentClient,
|
||||
GetCommand,
|
||||
PutCommand,
|
||||
DeleteCommand,
|
||||
TranslateConfig,
|
||||
} from '@aws-sdk/lib-dynamodb';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import { DYNAMODB_CONFIG } from './dynamodb.config';
|
||||
|
||||
export interface InputDocument {
|
||||
id: string;
|
||||
client_id: string;
|
||||
user_id: string;
|
||||
created_at: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
plugin: string;
|
||||
type: string;
|
||||
tables?: Array<{
|
||||
name: string;
|
||||
type: string;
|
||||
columns?: string[];
|
||||
reference_column?: { name: string; type: string };
|
||||
}>;
|
||||
credentials?: Record<string, any>;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DynamoDBService {
|
||||
private documentClient: DynamoDBDocumentClient;
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
|
||||
const dynamoConfig = { region: DYNAMODB_CONFIG.region() };
|
||||
const marshallOptions: TranslateConfig = {
|
||||
marshallOptions: {
|
||||
removeUndefinedValues: true,
|
||||
},
|
||||
};
|
||||
|
||||
const dynamoDb = new DynamoDBClient(dynamoConfig);
|
||||
this.documentClient = DynamoDBDocumentClient.from(dynamoDb, marshallOptions);
|
||||
}
|
||||
|
||||
async createInput(
|
||||
clientId: string,
|
||||
userId: string,
|
||||
data: {
|
||||
name: string;
|
||||
description?: string;
|
||||
plugin: string;
|
||||
type: string;
|
||||
tables?: Array<{
|
||||
name: string;
|
||||
type: string;
|
||||
columns?: string[];
|
||||
reference_column?: { name: string; type: string };
|
||||
}>;
|
||||
},
|
||||
): Promise<InputDocument> {
|
||||
const tableName = DYNAMODB_CONFIG.inputsTable();
|
||||
const id = uuid();
|
||||
const created_at = new Date().toISOString();
|
||||
|
||||
const item: InputDocument = {
|
||||
id,
|
||||
client_id: clientId,
|
||||
user_id: userId,
|
||||
created_at,
|
||||
name: data.name,
|
||||
description: data.description,
|
||||
plugin: data.plugin,
|
||||
type: data.type,
|
||||
tables: data.tables,
|
||||
};
|
||||
|
||||
this.logger.info('DynamoDB: Creating input', {
|
||||
tableName,
|
||||
inputId: id,
|
||||
plugin: data.plugin,
|
||||
});
|
||||
|
||||
const putCommand = new PutCommand({
|
||||
TableName: tableName,
|
||||
Item: item,
|
||||
});
|
||||
|
||||
try {
|
||||
await this.documentClient.send(putCommand);
|
||||
this.logger.info('DynamoDB: Input created successfully', { inputId: id });
|
||||
return item;
|
||||
} catch (error) {
|
||||
this.logger.error('DynamoDB: Failed to create input', {
|
||||
tableName,
|
||||
inputId: id,
|
||||
region: DYNAMODB_CONFIG.region(),
|
||||
error: error.message,
|
||||
errorName: error.name,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async findInput(clientId: string, inputId: string): Promise<InputDocument | null> {
|
||||
const tableName = DYNAMODB_CONFIG.inputsTable();
|
||||
|
||||
const getCommand = new GetCommand({
|
||||
TableName: tableName,
|
||||
Key: {
|
||||
id: inputId,
|
||||
client_id: clientId,
|
||||
},
|
||||
});
|
||||
|
||||
const { Item } = await this.documentClient.send(getCommand);
|
||||
|
||||
return Item as InputDocument | null;
|
||||
}
|
||||
|
||||
async deleteInput(clientId: string, inputId: string): Promise<void> {
|
||||
const tableName = DYNAMODB_CONFIG.inputsTable();
|
||||
|
||||
this.logger.info('DynamoDB: Deleting input', {
|
||||
tableName,
|
||||
inputId,
|
||||
});
|
||||
|
||||
const deleteCommand = new DeleteCommand({
|
||||
TableName: tableName,
|
||||
Key: {
|
||||
id: inputId,
|
||||
client_id: clientId,
|
||||
},
|
||||
});
|
||||
|
||||
await this.documentClient.send(deleteCommand);
|
||||
|
||||
this.logger.info('DynamoDB: Input deleted successfully', { inputId });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './dynamodb.service';
|
||||
export * from './dynamodb.module';
|
||||
export * from './dynamodb.config';
|
||||
@@ -0,0 +1,5 @@
|
||||
export const ELASTICSEARCH_CONFIG = {
|
||||
getUrl: () => process.env.ELASTICSEARCH_URL || 'http://localhost:9200',
|
||||
getApiKey: () => process.env.ELASTICSEARCH_API_KEY || '',
|
||||
timeout: 10000, // 10 seconds
|
||||
};
|
||||
@@ -0,0 +1,9 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { ElasticsearchService } from './elasticsearch.service';
|
||||
|
||||
@Module({
|
||||
providers: [ElasticsearchService, DadosferaLogger],
|
||||
exports: [ElasticsearchService],
|
||||
})
|
||||
export class ElasticsearchModule {}
|
||||
@@ -0,0 +1,375 @@
|
||||
import { Injectable, Inject } from '@nestjs/common';
|
||||
import axios, { AxiosInstance, AxiosError } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { ELASTICSEARCH_CONFIG } from './elasticsearch.config';
|
||||
|
||||
interface MultiLang {
|
||||
'en-us': string;
|
||||
'pt-br': string;
|
||||
'es-es': string;
|
||||
}
|
||||
|
||||
interface MultiLangArray {
|
||||
'en-us': string[];
|
||||
'pt-br': string[];
|
||||
'es-es': string[];
|
||||
}
|
||||
|
||||
interface ConnectorInfo {
|
||||
plugin: string;
|
||||
name: MultiLang;
|
||||
image: string;
|
||||
version: string;
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
interface PipelineDocument {
|
||||
id: string;
|
||||
name: MultiLang;
|
||||
description: MultiLang;
|
||||
customer_id: string;
|
||||
user_id: string;
|
||||
username: string;
|
||||
status: string;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
last_status_updated: string;
|
||||
tags: string[];
|
||||
// Connector metadata
|
||||
connection_id: string;
|
||||
connector_name: string;
|
||||
connector_plugin: string;
|
||||
connector_version: string;
|
||||
image_url: string;
|
||||
// Config
|
||||
config: {
|
||||
cron: string;
|
||||
tables?: string;
|
||||
};
|
||||
properties: string;
|
||||
type: string;
|
||||
in_use: number;
|
||||
keywords: MultiLangArray;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ElasticsearchService {
|
||||
private client: AxiosInstance;
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
this.client = axios.create({
|
||||
baseURL: ELASTICSEARCH_CONFIG.getUrl(),
|
||||
headers: {
|
||||
Authorization: `ApiKey ${ELASTICSEARCH_CONFIG.getApiKey()}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
timeout: ELASTICSEARCH_CONFIG.timeout,
|
||||
});
|
||||
}
|
||||
|
||||
private getIndex(customerName: string): string {
|
||||
return `${customerName}_pipelines`;
|
||||
}
|
||||
|
||||
private formatMultiLang(value: string): MultiLang {
|
||||
return {
|
||||
'en-us': value,
|
||||
'pt-br': value,
|
||||
'es-es': value,
|
||||
};
|
||||
}
|
||||
|
||||
private formatMultiLangArray(value: string[] = []): MultiLangArray {
|
||||
return {
|
||||
'en-us': value,
|
||||
'pt-br': value,
|
||||
'es-es': value,
|
||||
};
|
||||
}
|
||||
|
||||
buildPipelineDocument(
|
||||
pipelineId: string,
|
||||
data: {
|
||||
name: string;
|
||||
description?: string;
|
||||
user_id: string;
|
||||
username: string;
|
||||
customer_id: string;
|
||||
plugin: string;
|
||||
connection_id: string;
|
||||
cron?: string;
|
||||
tables?: string;
|
||||
properties?: Record<string, any>;
|
||||
type?: string;
|
||||
status?: string;
|
||||
created_at?: string;
|
||||
keywords?: string[];
|
||||
},
|
||||
connector: ConnectorInfo | null,
|
||||
): PipelineDocument {
|
||||
const now = new Date().toISOString();
|
||||
|
||||
return {
|
||||
id: pipelineId,
|
||||
connection_id: data.connection_id,
|
||||
connector_name: connector?.name?.['en-us'] || '',
|
||||
connector_plugin: connector?.plugin || data.plugin,
|
||||
connector_version: connector?.version || '1.0.0',
|
||||
created_at: data.created_at || now,
|
||||
updated_at: now,
|
||||
customer_id: data.customer_id,
|
||||
description: this.formatMultiLang(data.description || ''),
|
||||
image_url: connector?.image || '',
|
||||
keywords: this.formatMultiLangArray(data.keywords),
|
||||
name: this.formatMultiLang(data.name || ''),
|
||||
status: data.status || 'CREATED',
|
||||
user_id: data.user_id,
|
||||
username: data.username,
|
||||
config: {
|
||||
cron: data.cron,
|
||||
tables: data.tables,
|
||||
},
|
||||
properties: data.properties ? JSON.stringify(data.properties) : '{}',
|
||||
type: data.type,
|
||||
in_use: 1,
|
||||
last_status_updated: now,
|
||||
tags: connector?.tags || [],
|
||||
};
|
||||
}
|
||||
|
||||
async getConnectorByPlugin(plugin: string): Promise<ConnectorInfo | null> {
|
||||
this.logger.info('Elasticsearch: Looking up connector', { plugin });
|
||||
|
||||
try {
|
||||
const response = await this.client.post('/connectors/_search', {
|
||||
query: {
|
||||
term: { plugin: plugin },
|
||||
},
|
||||
size: 1,
|
||||
});
|
||||
|
||||
const hits = response.data.hits?.hits || [];
|
||||
if (hits.length === 0) {
|
||||
this.logger.warn('Elasticsearch: Connector not found', { plugin });
|
||||
return null;
|
||||
}
|
||||
|
||||
const source = hits[0]._source;
|
||||
return {
|
||||
plugin: source.plugin,
|
||||
name: source.name,
|
||||
image: source.image,
|
||||
version: source.version,
|
||||
tags: source.tags || [],
|
||||
};
|
||||
} catch (error) {
|
||||
this.handleError('getConnectorByPlugin', error, { plugin });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async createPipeline(
|
||||
customerName: string,
|
||||
pipelineId: string,
|
||||
data: {
|
||||
name: string;
|
||||
description?: string;
|
||||
user_id: string;
|
||||
username: string;
|
||||
customer_id: string;
|
||||
status?: string;
|
||||
created_at?: string;
|
||||
plugin: string;
|
||||
connection_id: string;
|
||||
cron?: string;
|
||||
tables?: string;
|
||||
properties?: Record<string, any>;
|
||||
type?: string;
|
||||
},
|
||||
connector: ConnectorInfo | null,
|
||||
): Promise<any> {
|
||||
const index = this.getIndex(customerName);
|
||||
const document = this.buildPipelineDocument(pipelineId, data, connector);
|
||||
|
||||
this.logger.info('Elasticsearch: Creating pipeline', {
|
||||
index,
|
||||
pipelineId,
|
||||
plugin: document.connector_plugin,
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.post(
|
||||
`${index}/_doc/${pipelineId}`,
|
||||
document,
|
||||
{ params: { refresh: 'wait_for' } },
|
||||
);
|
||||
|
||||
this.logger.info('Elasticsearch: Pipeline created successfully', {
|
||||
pipelineId,
|
||||
result: response.data.result,
|
||||
});
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.handleError('createPipeline', error, { pipelineId, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async updatePipeline(
|
||||
customerName: string,
|
||||
pipelineId: string,
|
||||
changes: {
|
||||
name?: string;
|
||||
description?: string;
|
||||
cron?: string;
|
||||
status?: string;
|
||||
tags?: string[];
|
||||
},
|
||||
): Promise<any> {
|
||||
const index = this.getIndex(customerName);
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const formattedChanges: Record<string, any> = {
|
||||
updated_at: now,
|
||||
};
|
||||
|
||||
if (changes.name !== undefined) {
|
||||
formattedChanges.name = this.formatMultiLang(changes.name);
|
||||
}
|
||||
|
||||
if (changes.description !== undefined) {
|
||||
formattedChanges.description = this.formatMultiLang(changes.description);
|
||||
}
|
||||
|
||||
if (changes.cron !== undefined) {
|
||||
// Update cron inside config object using ES script
|
||||
formattedChanges.config = { cron: changes.cron };
|
||||
}
|
||||
|
||||
if (changes.status !== undefined) {
|
||||
formattedChanges.status = changes.status;
|
||||
formattedChanges.last_status_updated = now;
|
||||
}
|
||||
|
||||
if (changes.tags !== undefined) {
|
||||
formattedChanges.tags = changes.tags;
|
||||
}
|
||||
|
||||
this.logger.info('Elasticsearch: Updating pipeline', {
|
||||
index,
|
||||
pipelineId,
|
||||
fields: Object.keys(formattedChanges),
|
||||
});
|
||||
|
||||
try {
|
||||
// Use script for partial config update to preserve other config fields
|
||||
const updateBody = changes.cron !== undefined
|
||||
? {
|
||||
script: {
|
||||
source: `
|
||||
ctx._source.updated_at = params.updated_at;
|
||||
if (params.name != null) ctx._source.name = params.name;
|
||||
if (params.description != null) ctx._source.description = params.description;
|
||||
if (params.cron != null) ctx._source.config.cron = params.cron;
|
||||
if (params.status != null) {
|
||||
ctx._source.status = params.status;
|
||||
ctx._source.last_status_updated = params.last_status_updated;
|
||||
}
|
||||
if (params.tags != null) ctx._source.tags = params.tags;
|
||||
`,
|
||||
params: {
|
||||
updated_at: now,
|
||||
name: changes.name !== undefined ? this.formatMultiLang(changes.name) : null,
|
||||
description: changes.description !== undefined ? this.formatMultiLang(changes.description) : null,
|
||||
cron: changes.cron || null,
|
||||
status: changes.status || null,
|
||||
last_status_updated: changes.status !== undefined ? now : null,
|
||||
tags: changes.tags || null,
|
||||
},
|
||||
},
|
||||
}
|
||||
: { doc: formattedChanges };
|
||||
|
||||
const response = await this.client.post(
|
||||
`${index}/_update/${pipelineId}`,
|
||||
updateBody,
|
||||
{ params: { refresh: 'wait_for' } },
|
||||
);
|
||||
|
||||
this.logger.info('Elasticsearch: Pipeline updated successfully', {
|
||||
pipelineId,
|
||||
result: response.data.result,
|
||||
});
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.handleError('updatePipeline', error, { pipelineId, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async deletePipeline(
|
||||
customerName: string,
|
||||
pipelineId: string,
|
||||
): Promise<any> {
|
||||
const index = this.getIndex(customerName);
|
||||
|
||||
this.logger.info('Elasticsearch: Deleting pipeline', {
|
||||
index,
|
||||
pipelineId,
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await this.client.delete(
|
||||
`${index}/_doc/${pipelineId}`,
|
||||
{ params: { refresh: 'wait_for' } },
|
||||
);
|
||||
|
||||
this.logger.info('Elasticsearch: Pipeline deleted successfully', {
|
||||
pipelineId,
|
||||
result: response.data.result,
|
||||
});
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
// If document not found, log warning but don't throw
|
||||
if (error instanceof AxiosError && error.response?.status === 404) {
|
||||
this.logger.warn('Elasticsearch: Pipeline not found for deletion', {
|
||||
pipelineId,
|
||||
index,
|
||||
});
|
||||
return { result: 'not_found' };
|
||||
}
|
||||
|
||||
this.handleError('deletePipeline', error, { pipelineId, index });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private handleError(
|
||||
operation: string,
|
||||
error: any,
|
||||
context: Record<string, any>,
|
||||
): void {
|
||||
if (error instanceof AxiosError) {
|
||||
this.logger.error(`Elasticsearch: ${operation} failed`, {
|
||||
...context,
|
||||
status: error.response?.status,
|
||||
statusText: error.response?.statusText,
|
||||
errorData: error.response?.data,
|
||||
message: error.message,
|
||||
});
|
||||
} else {
|
||||
this.logger.error(`Elasticsearch: ${operation} failed`, {
|
||||
...context,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export * from './elasticsearch.module';
|
||||
export * from './elasticsearch.service';
|
||||
export * from './elasticsearch.config';
|
||||
Reference in New Issue
Block a user