Merge branch 'beta' into feature/catalog-update

This commit is contained in:
vinicius gadea
2025-12-19 08:21:52 -03:00
committed by GitHub
12 changed files with 216 additions and 120 deletions
@@ -22,7 +22,7 @@ import { User, RequestUser } from '../../decorators/user.decorator';
import { PlatformApiService } from './platform-api.service';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { ElasticsearchService } from '../../services/elasticsearch';
import { DynamoDBService } from '../../services/dynamodb';
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
import { CustomersService } from '../customers/customers.service';
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
@@ -155,7 +155,7 @@ export class PlatformApiController {
* Extract and transform tables from jobs for DynamoDB input.
* Maps connector-specific fields to a common table format.
*
* - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name (reference_column)
* - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name/type (reference_column object)
* - Singer: type maps replication_method (FULL_TABLE -> full_load, INCREMENTAL -> incremental), no columns
* - S3: same mapping as Singer, no columns
*/
@@ -163,7 +163,7 @@ export class PlatformApiController {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
}> {
if (!jobs || jobs.length === 0) return [];
@@ -171,7 +171,7 @@ export class PlatformApiController {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
}> = [];
for (const job of jobs) {
@@ -179,12 +179,12 @@ export class PlatformApiController {
if (!input) continue;
if (connector === 'jdbc') {
// JDBC: table_name, load_type, column_include_list, incremental_column_name
// JDBC: table_name, load_type, column_include_list, incremental_column_name/type
const table: {
name: string;
type: string;
columns?: string[];
reference_column?: string;
reference_column?: ReferenceColumn;
} = {
name: input.table_name || '',
type: input.load_type || 'full_load',
@@ -193,8 +193,11 @@ export class PlatformApiController {
table.columns = input.column_include_list;
}
if (input.incremental_column_name) {
// reference_column is stored as a string (column name)
table.reference_column = input.incremental_column_name;
// reference_column is stored as an object with name and type
table.reference_column = {
name: input.incremental_column_name,
type: input.incremental_column_type || 'unknown',
};
}
tables.push(table);
} else if (connector === 'singer' || connector === 's3') {
@@ -317,11 +320,11 @@ export class PlatformApiController {
}
// Build changes for DynamoDB table entry
// reference_column is stored as a string (column name), not an object
// reference_column is stored as an object with name and type
const changes: {
type?: string;
columns?: string[];
reference_column?: string | null;
reference_column?: ReferenceColumn | null;
} = {};
@@ -332,8 +335,15 @@ export class PlatformApiController {
changes.columns = body.column_include_list;
}
if ('incremental_column_name' in body) {
// reference_column is just the column name as a string
changes.reference_column = body.incremental_column_name || null;
// reference_column is stored as an object with name and type
if (body.incremental_column_name) {
changes.reference_column = {
name: body.incremental_column_name,
type: body.incremental_column_type || 'unknown',
};
} else {
changes.reference_column = null;
}
}
// Update DynamoDB if there are changes
@@ -466,11 +476,13 @@ export class PlatformApiController {
});
}
const pipelineType = this.mapConnectorToDynamoType(connectorType);
this.logger.info('Syncing pipeline to Elasticsearch', {
customerName: user.customer_name,
pipelineId: body.id,
plugin,
connector: connectorType,
type: pipelineType,
properties,
inputId,
});
@@ -490,6 +502,7 @@ export class PlatformApiController {
cron: body.cron,
tables: inputId,
properties,
type: pipelineType,
},
connector,
);
@@ -617,21 +630,39 @@ export class PlatformApiController {
@ApiOperation({ summary: 'Execute a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async executePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/execute', user, body);
// Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/execute', user, enrichedBody);
}
@Post('pipeline/pause')
@ApiOperation({ summary: 'Pause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async pausePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/pause', user, body);
// Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/pause', user, enrichedBody);
}
@Post('pipeline/unpause')
@ApiOperation({ summary: 'Unpause a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async unpausePipeline(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, body);
// Inject customer_id (actually customer_name) into body for Platform-API
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
const enrichedBody = {
...body,
customer_id: user.customer_name,
};
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, enrichedBody);
}
@Put('pipeline/:pipelineId/memory')
@@ -691,18 +722,6 @@ export class PlatformApiController {
// ==================== PIPELINE RUN ROUTES ====================
@Post('pipeline/pipeline_run')
@ApiOperation({ summary: 'Create a pipeline run' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async createPipelineRun(@Body() body: any, @User() user: RequestUser) {
return this.platformApiService.proxy(
'POST',
'/pipeline/pipeline_run',
user,
body,
);
}
@Get('pipeline/:pipelineId/pipeline_run')
@ApiOperation({ summary: 'Get pipeline runs for a pipeline' })
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
@@ -711,9 +730,10 @@ export class PlatformApiController {
@User() user: RequestUser,
@Query() query: Record<string, string>,
) {
const normalizedId = this.normalizePipelineId(pipelineId);
return this.platformApiService.proxy(
'GET',
`/pipeline/${pipelineId}/pipeline_run`,
`/pipeline/${normalizedId}/pipeline_run`,
user,
undefined,
query,
@@ -728,9 +748,11 @@ export class PlatformApiController {
@Param('runId') runId: string,
@User() user: RequestUser,
) {
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
const normalizedRunId = this.normalizePipelineId(runId);
return this.platformApiService.proxy(
'GET',
`/pipeline/${pipelineId}/pipeline_run/${runId}`,
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}`,
user,
);
}
@@ -743,9 +765,10 @@ export class PlatformApiController {
@User() user: RequestUser,
@Query() query: Record<string, string>,
) {
const normalizedRunId = this.normalizePipelineId(runId);
return this.platformApiService.proxy(
'GET',
`/pipeline/pipeline_run/${runId}/logs`,
`/pipeline/pipeline_run/${normalizedRunId}/logs`,
user,
undefined,
query,
+22 -16
View File
@@ -229,27 +229,33 @@ export class RolesService {
const [roleTreated] = this.getRolesPermissionsName([role.role]);
return { role: roleTreated };
}
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
const newRoles: RoleDto[] = [];
for (const role of roles) {
const allPermissions = this.permissionsService.getAllPermissions(
this.language,
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
const newRole: RoleDto = {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};
const newRole: RoleDto = this.formatRole(role);
newRoles.push(newRole);
}
return newRoles;
}
private formatRole(role: GetRolesPermissionsName): RoleDto {
const allPermissions = this.permissionsService.getAllPermissions(
this.language
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
return {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};;
}
}
+12
View File
@@ -38,6 +38,12 @@ export class User {
department?: string;
@ApiProperty()
hierarchy?: string;
@ApiProperty()
bio?: string;
@ApiProperty()
companyName?: string;
@ApiProperty()
companySite?: string;
@ApiPropertyOptional()
customer?: Customer;
@ApiProperty()
@@ -109,6 +115,12 @@ export class UpdateUserReq {
@ApiPropertyOptional()
hierarchy?: string;
@ApiPropertyOptional()
bio?: string;
@ApiPropertyOptional()
companySite?: string;
@ApiPropertyOptional()
companyName?: string;
@ApiPropertyOptional()
roleNames?: string[];
}
+4 -1
View File
@@ -266,7 +266,6 @@ export class UsersController {
}
@Patch(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: UpdateUserRes })
async updateUser(
@User() user: RequestUser,
@@ -274,6 +273,10 @@ export class UsersController {
@Param('id') id: string,
@Language() language: LanguageEnum,
) {
if (user.user_id !== id || !user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
this.logger.info('updateUser', { user });
this.userService.setLanguage(language);
return await this.userService.updateUser(body, id, user.customer_id);
+10 -7
View File
@@ -148,20 +148,23 @@ export class UsersService implements OnModuleInit {
}
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
if (roleNames) {
const { roleNames, ...updateUserDTO } = req;
if (roleNames && roleNames.length > 0) {
await this.setRoles({ roleNames, userId: id }, customerId);
}
const { user } = await lastValueFrom(
this.usersClientService.UserUpdate({
name,
department: updateUserDTO.department,
email: updateUserDTO.email,
hierarchy: updateUserDTO.hierarchy,
jobTitle: updateUserDTO.jobTitle,
name: updateUserDTO.name,
bio: updateUserDTO.bio,
companyName: updateUserDTO.companyName,
companySite: updateUserDTO.companySite,
customerId,
id,
department,
hierarchy,
jobTitle,
email,
metabaseUserId: undefined,
}),
);