FEAT: add Platform API proxy with Elasticsearch and DynamoDB sync

- Add Platform API module to proxy requests to Platform API service
- Add Elasticsearch service for pipeline catalog sync (CRUD operations)
- Add DynamoDB service for storing pipeline tables/inputs configuration
- Sync pipeline creation/update/delete to Elasticsearch index
- Extract tables from jobs with connector-specific mappings:
  - JDBC: table_name, load_type, column_include_list, incremental_column_name
  - Singer: replication_method -> full_load/incremental
  - S3: same as Singer
- Map connector types to DynamoDB types (jdbc->database, singer->application, s3->file)
- Validate connector type is provided in job input
- Normalize pipeline IDs for Platform API (replace - with _), keep UUIDs for ES

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Rafael
2025-12-10 12:42:18 -03:00
co-authored by Claude Opus 4.5
parent e789076ed4
commit df674dd441
15 changed files with 3277 additions and 295 deletions
+4
View File
@@ -0,0 +1,4 @@
export const DYNAMODB_CONFIG = {
region: () => process.env.AWS_REGION || 'us-east-1',
inputsTable: () => process.env.INPUTS_DB || 'dadosfera-inputs-prd',
};
+9
View File
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { DynamoDBService } from './dynamodb.service';
@Module({
providers: [DynamoDBService, DadosferaLogger],
exports: [DynamoDBService],
})
export class DynamoDBModule {}
+148
View File
@@ -0,0 +1,148 @@
import { Injectable, Inject } from '@nestjs/common';
import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
import {
DynamoDBDocumentClient,
GetCommand,
PutCommand,
DeleteCommand,
TranslateConfig,
} from '@aws-sdk/lib-dynamodb';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { v4 as uuid } from 'uuid';
import { DYNAMODB_CONFIG } from './dynamodb.config';
export interface InputDocument {
id: string;
client_id: string;
user_id: string;
created_at: string;
name: string;
description?: string;
plugin: string;
type: string;
tables?: Array<{
name: string;
type: string;
columns?: string[];
reference_column?: { name: string; type: string };
}>;
credentials?: Record<string, any>;
}
@Injectable()
export class DynamoDBService {
private documentClient: DynamoDBDocumentClient;
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
const dynamoConfig = { region: DYNAMODB_CONFIG.region() };
const marshallOptions: TranslateConfig = {
marshallOptions: {
removeUndefinedValues: true,
},
};
const dynamoDb = new DynamoDBClient(dynamoConfig);
this.documentClient = DynamoDBDocumentClient.from(dynamoDb, marshallOptions);
}
async createInput(
clientId: string,
userId: string,
data: {
name: string;
description?: string;
plugin: string;
type: string;
tables?: Array<{
name: string;
type: string;
columns?: string[];
reference_column?: { name: string; type: string };
}>;
},
): Promise<InputDocument> {
const tableName = DYNAMODB_CONFIG.inputsTable();
const id = uuid();
const created_at = new Date().toISOString();
const item: InputDocument = {
id,
client_id: clientId,
user_id: userId,
created_at,
name: data.name,
description: data.description,
plugin: data.plugin,
type: data.type,
tables: data.tables,
};
this.logger.info('DynamoDB: Creating input', {
tableName,
inputId: id,
plugin: data.plugin,
});
const putCommand = new PutCommand({
TableName: tableName,
Item: item,
});
try {
await this.documentClient.send(putCommand);
this.logger.info('DynamoDB: Input created successfully', { inputId: id });
return item;
} catch (error) {
this.logger.error('DynamoDB: Failed to create input', {
tableName,
inputId: id,
region: DYNAMODB_CONFIG.region(),
error: error.message,
errorName: error.name,
});
throw error;
}
}
async findInput(clientId: string, inputId: string): Promise<InputDocument | null> {
const tableName = DYNAMODB_CONFIG.inputsTable();
const getCommand = new GetCommand({
TableName: tableName,
Key: {
id: inputId,
client_id: clientId,
},
});
const { Item } = await this.documentClient.send(getCommand);
return Item as InputDocument | null;
}
async deleteInput(clientId: string, inputId: string): Promise<void> {
const tableName = DYNAMODB_CONFIG.inputsTable();
this.logger.info('DynamoDB: Deleting input', {
tableName,
inputId,
});
const deleteCommand = new DeleteCommand({
TableName: tableName,
Key: {
id: inputId,
client_id: clientId,
},
});
await this.documentClient.send(deleteCommand);
this.logger.info('DynamoDB: Input deleted successfully', { inputId });
}
}
+3
View File
@@ -0,0 +1,3 @@
export * from './dynamodb.service';
export * from './dynamodb.module';
export * from './dynamodb.config';
@@ -0,0 +1,5 @@
export const ELASTICSEARCH_CONFIG = {
getUrl: () => process.env.ELASTICSEARCH_URL || 'http://localhost:9200',
getApiKey: () => process.env.ELASTICSEARCH_API_KEY || '',
timeout: 10000, // 10 seconds
};
@@ -0,0 +1,9 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ElasticsearchService } from './elasticsearch.service';
@Module({
providers: [ElasticsearchService, DadosferaLogger],
exports: [ElasticsearchService],
})
export class ElasticsearchModule {}
@@ -0,0 +1,375 @@
import { Injectable, Inject } from '@nestjs/common';
import axios, { AxiosInstance, AxiosError } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ELASTICSEARCH_CONFIG } from './elasticsearch.config';
interface MultiLang {
'en-us': string;
'pt-br': string;
'es-es': string;
}
interface MultiLangArray {
'en-us': string[];
'pt-br': string[];
'es-es': string[];
}
interface ConnectorInfo {
plugin: string;
name: MultiLang;
image: string;
version: string;
tags: string[];
}
interface PipelineDocument {
id: string;
name: MultiLang;
description: MultiLang;
customer_id: string;
user_id: string;
username: string;
status: string;
created_at: string;
updated_at: string;
last_status_updated: string;
tags: string[];
// Connector metadata
connection_id: string;
connector_name: string;
connector_plugin: string;
connector_version: string;
image_url: string;
// Config
config: {
cron: string;
tables?: string;
};
properties: string;
type: string;
in_use: number;
keywords: MultiLangArray;
}
@Injectable()
export class ElasticsearchService {
private client: AxiosInstance;
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
this.client = axios.create({
baseURL: ELASTICSEARCH_CONFIG.getUrl(),
headers: {
Authorization: `ApiKey ${ELASTICSEARCH_CONFIG.getApiKey()}`,
'Content-Type': 'application/json',
},
timeout: ELASTICSEARCH_CONFIG.timeout,
});
}
private getIndex(customerName: string): string {
return `${customerName}_pipelines`;
}
private formatMultiLang(value: string): MultiLang {
return {
'en-us': value,
'pt-br': value,
'es-es': value,
};
}
private formatMultiLangArray(value: string[] = []): MultiLangArray {
return {
'en-us': value,
'pt-br': value,
'es-es': value,
};
}
buildPipelineDocument(
pipelineId: string,
data: {
name: string;
description?: string;
user_id: string;
username: string;
customer_id: string;
plugin: string;
connection_id: string;
cron?: string;
tables?: string;
properties?: Record<string, any>;
type?: string;
status?: string;
created_at?: string;
keywords?: string[];
},
connector: ConnectorInfo | null,
): PipelineDocument {
const now = new Date().toISOString();
return {
id: pipelineId,
connection_id: data.connection_id,
connector_name: connector?.name?.['en-us'] || '',
connector_plugin: connector?.plugin || data.plugin,
connector_version: connector?.version || '1.0.0',
created_at: data.created_at || now,
updated_at: now,
customer_id: data.customer_id,
description: this.formatMultiLang(data.description || ''),
image_url: connector?.image || '',
keywords: this.formatMultiLangArray(data.keywords),
name: this.formatMultiLang(data.name || ''),
status: data.status || 'CREATED',
user_id: data.user_id,
username: data.username,
config: {
cron: data.cron,
tables: data.tables,
},
properties: data.properties ? JSON.stringify(data.properties) : '{}',
type: data.type,
in_use: 1,
last_status_updated: now,
tags: connector?.tags || [],
};
}
async getConnectorByPlugin(plugin: string): Promise<ConnectorInfo | null> {
this.logger.info('Elasticsearch: Looking up connector', { plugin });
try {
const response = await this.client.post('/connectors/_search', {
query: {
term: { plugin: plugin },
},
size: 1,
});
const hits = response.data.hits?.hits || [];
if (hits.length === 0) {
this.logger.warn('Elasticsearch: Connector not found', { plugin });
return null;
}
const source = hits[0]._source;
return {
plugin: source.plugin,
name: source.name,
image: source.image,
version: source.version,
tags: source.tags || [],
};
} catch (error) {
this.handleError('getConnectorByPlugin', error, { plugin });
return null;
}
}
async createPipeline(
customerName: string,
pipelineId: string,
data: {
name: string;
description?: string;
user_id: string;
username: string;
customer_id: string;
status?: string;
created_at?: string;
plugin: string;
connection_id: string;
cron?: string;
tables?: string;
properties?: Record<string, any>;
type?: string;
},
connector: ConnectorInfo | null,
): Promise<any> {
const index = this.getIndex(customerName);
const document = this.buildPipelineDocument(pipelineId, data, connector);
this.logger.info('Elasticsearch: Creating pipeline', {
index,
pipelineId,
plugin: document.connector_plugin,
});
try {
const response = await this.client.post(
`${index}/_doc/${pipelineId}`,
document,
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Pipeline created successfully', {
pipelineId,
result: response.data.result,
});
return response.data;
} catch (error) {
this.handleError('createPipeline', error, { pipelineId, index });
throw error;
}
}
async updatePipeline(
customerName: string,
pipelineId: string,
changes: {
name?: string;
description?: string;
cron?: string;
status?: string;
tags?: string[];
},
): Promise<any> {
const index = this.getIndex(customerName);
const now = new Date().toISOString();
const formattedChanges: Record<string, any> = {
updated_at: now,
};
if (changes.name !== undefined) {
formattedChanges.name = this.formatMultiLang(changes.name);
}
if (changes.description !== undefined) {
formattedChanges.description = this.formatMultiLang(changes.description);
}
if (changes.cron !== undefined) {
// Update cron inside config object using ES script
formattedChanges.config = { cron: changes.cron };
}
if (changes.status !== undefined) {
formattedChanges.status = changes.status;
formattedChanges.last_status_updated = now;
}
if (changes.tags !== undefined) {
formattedChanges.tags = changes.tags;
}
this.logger.info('Elasticsearch: Updating pipeline', {
index,
pipelineId,
fields: Object.keys(formattedChanges),
});
try {
// Use script for partial config update to preserve other config fields
const updateBody = changes.cron !== undefined
? {
script: {
source: `
ctx._source.updated_at = params.updated_at;
if (params.name != null) ctx._source.name = params.name;
if (params.description != null) ctx._source.description = params.description;
if (params.cron != null) ctx._source.config.cron = params.cron;
if (params.status != null) {
ctx._source.status = params.status;
ctx._source.last_status_updated = params.last_status_updated;
}
if (params.tags != null) ctx._source.tags = params.tags;
`,
params: {
updated_at: now,
name: changes.name !== undefined ? this.formatMultiLang(changes.name) : null,
description: changes.description !== undefined ? this.formatMultiLang(changes.description) : null,
cron: changes.cron || null,
status: changes.status || null,
last_status_updated: changes.status !== undefined ? now : null,
tags: changes.tags || null,
},
},
}
: { doc: formattedChanges };
const response = await this.client.post(
`${index}/_update/${pipelineId}`,
updateBody,
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Pipeline updated successfully', {
pipelineId,
result: response.data.result,
});
return response.data;
} catch (error) {
this.handleError('updatePipeline', error, { pipelineId, index });
throw error;
}
}
async deletePipeline(
customerName: string,
pipelineId: string,
): Promise<any> {
const index = this.getIndex(customerName);
this.logger.info('Elasticsearch: Deleting pipeline', {
index,
pipelineId,
});
try {
const response = await this.client.delete(
`${index}/_doc/${pipelineId}`,
{ params: { refresh: 'wait_for' } },
);
this.logger.info('Elasticsearch: Pipeline deleted successfully', {
pipelineId,
result: response.data.result,
});
return response.data;
} catch (error) {
// If document not found, log warning but don't throw
if (error instanceof AxiosError && error.response?.status === 404) {
this.logger.warn('Elasticsearch: Pipeline not found for deletion', {
pipelineId,
index,
});
return { result: 'not_found' };
}
this.handleError('deletePipeline', error, { pipelineId, index });
throw error;
}
}
private handleError(
operation: string,
error: any,
context: Record<string, any>,
): void {
if (error instanceof AxiosError) {
this.logger.error(`Elasticsearch: ${operation} failed`, {
...context,
status: error.response?.status,
statusText: error.response?.statusText,
errorData: error.response?.data,
message: error.message,
});
} else {
this.logger.error(`Elasticsearch: ${operation} failed`, {
...context,
message: error.message,
stack: error.stack,
});
}
}
}
+3
View File
@@ -0,0 +1,3 @@
export * from './elasticsearch.module';
export * from './elasticsearch.service';
export * from './elasticsearch.config';