Compare commits

...
Author SHA1 Message Date
RafaelandClaude Opus 4.5 f61c241dde FIX: inject customer_id in pipeline execute/pause/unpause routes
Users no longer need to provide customer_id in the request body for
execute, pause, and unpause pipeline operations - it's now automatically
injected from the authenticated user's session.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-18 10:30:09 -03:00
RafaelandClaude Opus 4.5 9c55c22230 FIX: store reference_column as object with name and type
The protobuf definition expects reference_column to be an object with
name and type fields, but it was being stored as just a string (column
name). This caused pipeline fetching to fail with the error:
".NewTable.reference_column: object expected"

Changes:
- Update ReferenceColumn interface in DynamoDB service
- Update extractTablesFromJobs to create reference_column object
- Update syncJobInputToDynamoDB to handle reference_column object

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 13:47:57 -03:00
RafaelandClaude Opus 4.5 fb521f53cd FIX: pass type field to Elasticsearch createPipeline
The type field was missing from the createPipeline call,
causing ES documents to not have the type field set.

Maps: jdbc->database, singer->application, s3->file

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 21:09:07 -03:00
Rafael acb631e33d UPDATE: force deployment 2025-12-16 19:16:35 -03:00
RafaelandClaude Opus 4.5 e616061c21 fix: normalize IDs in pipeline run routes before calling Platform API
- Add normalization for pipelineId and runId in getPipelineRuns, getPipelineRun, and getPipelineRunLogs
- Remove unused createPipelineRun route

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-16 19:10:23 -03:00
3 changed files with 61 additions and 34 deletions
-1
View File
@@ -2,7 +2,6 @@
<image src="./assets/maestro.svg" style="width:10rem"> <image src="./assets/maestro.svg" style="width:10rem">
</p> </p>
# Maestro # Maestro
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços. Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
@@ -22,7 +22,7 @@ import { User, RequestUser } from '../../decorators/user.decorator';
import { PlatformApiService } from './platform-api.service'; import { PlatformApiService } from './platform-api.service';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { ElasticsearchService } from '../../services/elasticsearch'; import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService } from '../../services/dynamodb'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service'; import { CustomersService } from '../customers/customers.service';
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
@@ -155,7 +155,7 @@ export class PlatformApiController {
* Extract and transform tables from jobs for DynamoDB input. * Extract and transform tables from jobs for DynamoDB input.
* Maps connector-specific fields to a common table format. * Maps connector-specific fields to a common table format.
* *
* - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name (reference_column) * - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name/type (reference_column object)
* - Singer: type maps replication_method (FULL_TABLE -> full_load, INCREMENTAL -> incremental), no columns * - Singer: type maps replication_method (FULL_TABLE -> full_load, INCREMENTAL -> incremental), no columns
* - S3: same mapping as Singer, no columns * - S3: same mapping as Singer, no columns
*/ */
@@ -163,7 +163,7 @@ export class PlatformApiController {
name: string; name: string;
type: string; type: string;
columns?: string[]; columns?: string[];
reference_column?: string; reference_column?: ReferenceColumn;
}> { }> {
if (!jobs || jobs.length === 0) return []; if (!jobs || jobs.length === 0) return [];
@@ -171,7 +171,7 @@ export class PlatformApiController {
name: string; name: string;
type: string; type: string;
columns?: string[]; columns?: string[];
reference_column?: string; reference_column?: ReferenceColumn;
}> = []; }> = [];
for (const job of jobs) { for (const job of jobs) {
@@ -179,12 +179,12 @@ export class PlatformApiController {
if (!input) continue; if (!input) continue;
if (connector === 'jdbc') { if (connector === 'jdbc') {
// JDBC: table_name, load_type, column_include_list, incremental_column_name // JDBC: table_name, load_type, column_include_list, incremental_column_name/type
const table: { const table: {
name: string; name: string;
type: string; type: string;
columns?: string[]; columns?: string[];
reference_column?: string; reference_column?: ReferenceColumn;
} = { } = {
name: input.table_name || '', name: input.table_name || '',
type: input.load_type || 'full_load', type: input.load_type || 'full_load',
@@ -193,8 +193,11 @@ export class PlatformApiController {
table.columns = input.column_include_list; table.columns = input.column_include_list;
} }
if (input.incremental_column_name) { if (input.incremental_column_name) {
// reference_column is stored as a string (column name) // reference_column is stored as an object with name and type
table.reference_column = input.incremental_column_name; table.reference_column = {
name: input.incremental_column_name,
type: input.incremental_column_type || 'unknown',
};
} }
tables.push(table); tables.push(table);
} else if (connector === 'singer' || connector === 's3') { } else if (connector === 'singer' || connector === 's3') {
@@ -317,11 +320,11 @@ export class PlatformApiController {
} }
// Build changes for DynamoDB table entry // Build changes for DynamoDB table entry
// reference_column is stored as a string (column name), not an object // reference_column is stored as an object with name and type
const changes: { const changes: {
type?: string; type?: string;
columns?: string[]; columns?: string[];
reference_column?: string | null; reference_column?: ReferenceColumn | null;
} = {}; } = {};
@@ -332,8 +335,15 @@ export class PlatformApiController {
changes.columns = body.column_include_list; changes.columns = body.column_include_list;
} }
if ('incremental_column_name' in body) { if ('incremental_column_name' in body) {
// reference_column is just the column name as a string // reference_column is stored as an object with name and type
changes.reference_column = body.incremental_column_name || null; if (body.incremental_column_name) {
changes.reference_column = {
name: body.incremental_column_name,
type: body.incremental_column_type || 'unknown',
};
} else {
changes.reference_column = null;
}
} }
// Update DynamoDB if there are changes // Update DynamoDB if there are changes
@@ -466,11 +476,13 @@ export class PlatformApiController {
}); });
} }
const pipelineType = this.mapConnectorToDynamoType(connectorType);
this.logger.info('Syncing pipeline to Elasticsearch', { this.logger.info('Syncing pipeline to Elasticsearch', {
customerName: user.customer_name, customerName: user.customer_name,
pipelineId: body.id, pipelineId: body.id,
plugin, plugin,
connector: connectorType, connector: connectorType,
type: pipelineType,
properties, properties,
inputId, inputId,
}); });
@@ -490,6 +502,7 @@ export class PlatformApiController {
cron: body.cron, cron: body.cron,
tables: inputId, tables: inputId,
properties, properties,
type: pipelineType,
}, },
connector, connector,
); );
@@ -617,21 +630,39 @@ export class PlatformApiController {
@ApiOperation({ summary: 'Execute a pipeline' }) @ApiOperation({ summary: 'Execute a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async executePipeline(@Body() body: any, @User() user: RequestUser) { async executePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/execute', user, body); // Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/execute', user, enrichedBody);
} }
@Post('pipeline/pause') @Post('pipeline/pause')
@ApiOperation({ summary: 'Pause a pipeline' }) @ApiOperation({ summary: 'Pause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async pausePipeline(@Body() body: any, @User() user: RequestUser) { async pausePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/pause', user, body); // Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/pause', user, enrichedBody);
} }
@Post('pipeline/unpause') @Post('pipeline/unpause')
@ApiOperation({ summary: 'Unpause a pipeline' }) @ApiOperation({ summary: 'Unpause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async unpausePipeline(@Body() body: any, @User() user: RequestUser) { async unpausePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, body); // Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, enrichedBody);
} }
@Put('pipeline/:pipelineId/memory') @Put('pipeline/:pipelineId/memory')
@@ -691,18 +722,6 @@ export class PlatformApiController {
// ==================== PIPELINE RUN ROUTES ==================== // ==================== PIPELINE RUN ROUTES ====================
@Post('pipeline/pipeline_run')
@ApiOperation({ summary: 'Create a pipeline run' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async createPipelineRun(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy(
'POST',
'/pipeline/pipeline_run',
user,
body,
);
}
@Get('pipeline/:pipelineId/pipeline_run') @Get('pipeline/:pipelineId/pipeline_run')
@ApiOperation({ summary: 'Get pipeline runs for a pipeline' }) @ApiOperation({ summary: 'Get pipeline runs for a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
@@ -711,9 +730,10 @@ export class PlatformApiController {
@User() user: RequestUser, @User() user: RequestUser,
@Query() query: Record<string, string>, @Query() query: Record<string, string>,
) { ) {
const normalizedId = this.normalizePipelineId(pipelineId);
return this.platformApiService.proxy( return this.platformApiService.proxy(
'GET', 'GET',
`/pipeline/${pipelineId}/pipeline_run`, `/pipeline/${normalizedId}/pipeline_run`,
user, user,
undefined, undefined,
query, query,
@@ -728,9 +748,11 @@ export class PlatformApiController {
@Param('runId') runId: string, @Param('runId') runId: string,
@User() user: RequestUser, @User() user: RequestUser,
) { ) {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const normalizedRunId = this.normalizePipelineId(runId);
return this.platformApiService.proxy( return this.platformApiService.proxy(
'GET', 'GET',
`/pipeline/${pipelineId}/pipeline_run/${runId}`, `/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}`,
user, user,
); );
} }
@@ -743,9 +765,10 @@ export class PlatformApiController {
@User() user: RequestUser, @User() user: RequestUser,
@Query() query: Record<string, string>, @Query() query: Record<string, string>,
) { ) {
const normalizedRunId = this.normalizePipelineId(runId);
return this.platformApiService.proxy( return this.platformApiService.proxy(
'GET', 'GET',
`/pipeline/pipeline_run/${runId}/logs`, `/pipeline/pipeline_run/${normalizedRunId}/logs`,
user, user,
undefined, undefined,
query, query,
+8 -3
View File
@@ -11,6 +11,11 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { v4 as uuid } from 'uuid'; import { v4 as uuid } from 'uuid';
import { DYNAMODB_CONFIG } from './dynamodb.config'; import { DYNAMODB_CONFIG } from './dynamodb.config';
export interface ReferenceColumn {
name: string;
type: string;
}
export interface InputDocument { export interface InputDocument {
id: string; id: string;
client_id: string; client_id: string;
@@ -24,7 +29,7 @@ export interface InputDocument {
name: string; name: string;
type: string; type: string;
columns?: string[]; columns?: string[];
reference_column?: string; reference_column?: ReferenceColumn;
}>; }>;
credentials?: Record<string, any>; credentials?: Record<string, any>;
} }
@@ -62,7 +67,7 @@ export class DynamoDBService {
name: string; name: string;
type: string; type: string;
columns?: string[]; columns?: string[];
reference_column?: string; reference_column?: ReferenceColumn;
}>; }>;
}, },
): Promise<InputDocument> { ): Promise<InputDocument> {
@@ -157,7 +162,7 @@ export class DynamoDBService {
changes: { changes: {
type?: string; type?: string;
columns?: string[]; columns?: string[];
reference_column?: string | null; reference_column?: ReferenceColumn | null;
}, },
): Promise<void> { ): Promise<void> {
const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); const dynamoTableName = DYNAMODB_CONFIG.inputsTable();