mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-20 20:49:07 +00:00
Merge branch 'beta' into feature/catalog-update
This commit is contained in:
@@ -22,7 +22,7 @@ import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { PlatformApiService } from './platform-api.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { ElasticsearchService } from '../../services/elasticsearch';
|
||||
import { DynamoDBService } from '../../services/dynamodb';
|
||||
import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb';
|
||||
import { CustomersService } from '../customers/customers.service';
|
||||
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
||||
|
||||
@@ -155,7 +155,7 @@ export class PlatformApiController {
|
||||
* Extract and transform tables from jobs for DynamoDB input.
|
||||
* Maps connector-specific fields to a common table format.
|
||||
*
|
||||
* - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name (reference_column)
|
||||
* - JDBC: load_type, table_name, column_include_list (columns), incremental_column_name/type (reference_column object)
|
||||
* - Singer: type maps replication_method (FULL_TABLE -> full_load, INCREMENTAL -> incremental), no columns
|
||||
* - S3: same mapping as Singer, no columns
|
||||
*/
|
||||
@@ -163,7 +163,7 @@ export class PlatformApiController {
|
||||
name: string;
|
||||
type: string;
|
||||
columns?: string[];
|
||||
reference_column?: string;
|
||||
reference_column?: ReferenceColumn;
|
||||
}> {
|
||||
if (!jobs || jobs.length === 0) return [];
|
||||
|
||||
@@ -171,7 +171,7 @@ export class PlatformApiController {
|
||||
name: string;
|
||||
type: string;
|
||||
columns?: string[];
|
||||
reference_column?: string;
|
||||
reference_column?: ReferenceColumn;
|
||||
}> = [];
|
||||
|
||||
for (const job of jobs) {
|
||||
@@ -179,12 +179,12 @@ export class PlatformApiController {
|
||||
if (!input) continue;
|
||||
|
||||
if (connector === 'jdbc') {
|
||||
// JDBC: table_name, load_type, column_include_list, incremental_column_name
|
||||
// JDBC: table_name, load_type, column_include_list, incremental_column_name/type
|
||||
const table: {
|
||||
name: string;
|
||||
type: string;
|
||||
columns?: string[];
|
||||
reference_column?: string;
|
||||
reference_column?: ReferenceColumn;
|
||||
} = {
|
||||
name: input.table_name || '',
|
||||
type: input.load_type || 'full_load',
|
||||
@@ -193,8 +193,11 @@ export class PlatformApiController {
|
||||
table.columns = input.column_include_list;
|
||||
}
|
||||
if (input.incremental_column_name) {
|
||||
// reference_column is stored as a string (column name)
|
||||
table.reference_column = input.incremental_column_name;
|
||||
// reference_column is stored as an object with name and type
|
||||
table.reference_column = {
|
||||
name: input.incremental_column_name,
|
||||
type: input.incremental_column_type || 'unknown',
|
||||
};
|
||||
}
|
||||
tables.push(table);
|
||||
} else if (connector === 'singer' || connector === 's3') {
|
||||
@@ -317,11 +320,11 @@ export class PlatformApiController {
|
||||
}
|
||||
|
||||
// Build changes for DynamoDB table entry
|
||||
// reference_column is stored as a string (column name), not an object
|
||||
// reference_column is stored as an object with name and type
|
||||
const changes: {
|
||||
type?: string;
|
||||
columns?: string[];
|
||||
reference_column?: string | null;
|
||||
reference_column?: ReferenceColumn | null;
|
||||
} = {};
|
||||
|
||||
|
||||
@@ -332,8 +335,15 @@ export class PlatformApiController {
|
||||
changes.columns = body.column_include_list;
|
||||
}
|
||||
if ('incremental_column_name' in body) {
|
||||
// reference_column is just the column name as a string
|
||||
changes.reference_column = body.incremental_column_name || null;
|
||||
// reference_column is stored as an object with name and type
|
||||
if (body.incremental_column_name) {
|
||||
changes.reference_column = {
|
||||
name: body.incremental_column_name,
|
||||
type: body.incremental_column_type || 'unknown',
|
||||
};
|
||||
} else {
|
||||
changes.reference_column = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Update DynamoDB if there are changes
|
||||
@@ -466,11 +476,13 @@ export class PlatformApiController {
|
||||
});
|
||||
}
|
||||
|
||||
const pipelineType = this.mapConnectorToDynamoType(connectorType);
|
||||
this.logger.info('Syncing pipeline to Elasticsearch', {
|
||||
customerName: user.customer_name,
|
||||
pipelineId: body.id,
|
||||
plugin,
|
||||
connector: connectorType,
|
||||
type: pipelineType,
|
||||
properties,
|
||||
inputId,
|
||||
});
|
||||
@@ -490,6 +502,7 @@ export class PlatformApiController {
|
||||
cron: body.cron,
|
||||
tables: inputId,
|
||||
properties,
|
||||
type: pipelineType,
|
||||
},
|
||||
connector,
|
||||
);
|
||||
@@ -617,21 +630,39 @@ export class PlatformApiController {
|
||||
@ApiOperation({ summary: 'Execute a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async executePipeline(@Body() body: any, @User() user: RequestUser) {
|
||||
return this.platformApiService.proxy('POST', '/pipeline/execute', user, body);
|
||||
// Inject customer_id (actually customer_name) into body for Platform-API
|
||||
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
|
||||
const enrichedBody = {
|
||||
...body,
|
||||
customer_id: user.customer_name,
|
||||
};
|
||||
return this.platformApiService.proxy('POST', '/pipeline/execute', user, enrichedBody);
|
||||
}
|
||||
|
||||
@Post('pipeline/pause')
|
||||
@ApiOperation({ summary: 'Pause a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async pausePipeline(@Body() body: any, @User() user: RequestUser) {
|
||||
return this.platformApiService.proxy('POST', '/pipeline/pause', user, body);
|
||||
// Inject customer_id (actually customer_name) into body for Platform-API
|
||||
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
|
||||
const enrichedBody = {
|
||||
...body,
|
||||
customer_id: user.customer_name,
|
||||
};
|
||||
return this.platformApiService.proxy('POST', '/pipeline/pause', user, enrichedBody);
|
||||
}
|
||||
|
||||
@Post('pipeline/unpause')
|
||||
@ApiOperation({ summary: 'Unpause a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async unpausePipeline(@Body() body: any, @User() user: RequestUser) {
|
||||
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, body);
|
||||
// Inject customer_id (actually customer_name) into body for Platform-API
|
||||
// Note: Platform-API was created before customer_id existed, so it expects customer_name in the customer_id field
|
||||
const enrichedBody = {
|
||||
...body,
|
||||
customer_id: user.customer_name,
|
||||
};
|
||||
return this.platformApiService.proxy('POST', '/pipeline/unpause', user, enrichedBody);
|
||||
}
|
||||
|
||||
@Put('pipeline/:pipelineId/memory')
|
||||
@@ -691,18 +722,6 @@ export class PlatformApiController {
|
||||
|
||||
// ==================== PIPELINE RUN ROUTES ====================
|
||||
|
||||
@Post('pipeline/pipeline_run')
|
||||
@ApiOperation({ summary: 'Create a pipeline run' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async createPipelineRun(@Body() body: any, @User() user: RequestUser) {
|
||||
return this.platformApiService.proxy(
|
||||
'POST',
|
||||
'/pipeline/pipeline_run',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@Get('pipeline/:pipelineId/pipeline_run')
|
||||
@ApiOperation({ summary: 'Get pipeline runs for a pipeline' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
@@ -711,9 +730,10 @@ export class PlatformApiController {
|
||||
@User() user: RequestUser,
|
||||
@Query() query: Record<string, string>,
|
||||
) {
|
||||
const normalizedId = this.normalizePipelineId(pipelineId);
|
||||
return this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/pipeline/${pipelineId}/pipeline_run`,
|
||||
`/pipeline/${normalizedId}/pipeline_run`,
|
||||
user,
|
||||
undefined,
|
||||
query,
|
||||
@@ -728,9 +748,11 @@ export class PlatformApiController {
|
||||
@Param('runId') runId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const normalizedRunId = this.normalizePipelineId(runId);
|
||||
return this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/pipeline/${pipelineId}/pipeline_run/${runId}`,
|
||||
`/pipeline/${normalizedPipelineId}/pipeline_run/${normalizedRunId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
@@ -743,9 +765,10 @@ export class PlatformApiController {
|
||||
@User() user: RequestUser,
|
||||
@Query() query: Record<string, string>,
|
||||
) {
|
||||
const normalizedRunId = this.normalizePipelineId(runId);
|
||||
return this.platformApiService.proxy(
|
||||
'GET',
|
||||
`/pipeline/pipeline_run/${runId}/logs`,
|
||||
`/pipeline/pipeline_run/${normalizedRunId}/logs`,
|
||||
user,
|
||||
undefined,
|
||||
query,
|
||||
|
||||
@@ -229,27 +229,33 @@ export class RolesService {
|
||||
const [roleTreated] = this.getRolesPermissionsName([role.role]);
|
||||
return { role: roleTreated };
|
||||
}
|
||||
|
||||
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
|
||||
const newRoles: RoleDto[] = [];
|
||||
for (const role of roles) {
|
||||
const allPermissions = this.permissionsService.getAllPermissions(
|
||||
this.language,
|
||||
);
|
||||
const newPermissions = role.permissions.map((p) => {
|
||||
const permission = allPermissions.find((per) => per.seqid === p.seqid);
|
||||
return {
|
||||
...p,
|
||||
name: permission.name,
|
||||
id: p.seqid,
|
||||
};
|
||||
});
|
||||
const newRole: RoleDto = {
|
||||
...role,
|
||||
permissions: newPermissions,
|
||||
isPublic: role.isPublic,
|
||||
};
|
||||
const newRole: RoleDto = this.formatRole(role);
|
||||
newRoles.push(newRole);
|
||||
}
|
||||
return newRoles;
|
||||
}
|
||||
|
||||
private formatRole(role: GetRolesPermissionsName): RoleDto {
|
||||
const allPermissions = this.permissionsService.getAllPermissions(
|
||||
this.language
|
||||
);
|
||||
const newPermissions = role.permissions.map((p) => {
|
||||
const permission = allPermissions.find((per) => per.seqid === p.seqid);
|
||||
return {
|
||||
...p,
|
||||
name: permission.name,
|
||||
id: p.seqid,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
...role,
|
||||
permissions: newPermissions,
|
||||
isPublic: role.isPublic,
|
||||
};;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,12 @@ export class User {
|
||||
department?: string;
|
||||
@ApiProperty()
|
||||
hierarchy?: string;
|
||||
@ApiProperty()
|
||||
bio?: string;
|
||||
@ApiProperty()
|
||||
companyName?: string;
|
||||
@ApiProperty()
|
||||
companySite?: string;
|
||||
@ApiPropertyOptional()
|
||||
customer?: Customer;
|
||||
@ApiProperty()
|
||||
@@ -109,6 +115,12 @@ export class UpdateUserReq {
|
||||
@ApiPropertyOptional()
|
||||
hierarchy?: string;
|
||||
@ApiPropertyOptional()
|
||||
bio?: string;
|
||||
@ApiPropertyOptional()
|
||||
companySite?: string;
|
||||
@ApiPropertyOptional()
|
||||
companyName?: string;
|
||||
@ApiPropertyOptional()
|
||||
roleNames?: string[];
|
||||
}
|
||||
|
||||
|
||||
@@ -266,7 +266,6 @@ export class UsersController {
|
||||
}
|
||||
|
||||
@Patch(':id')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
@ApiOkResponse({ type: UpdateUserRes })
|
||||
async updateUser(
|
||||
@User() user: RequestUser,
|
||||
@@ -274,6 +273,10 @@ export class UsersController {
|
||||
@Param('id') id: string,
|
||||
@Language() language: LanguageEnum,
|
||||
) {
|
||||
|
||||
if (user.user_id !== id || !user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)) {
|
||||
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
|
||||
}
|
||||
this.logger.info('updateUser', { user });
|
||||
this.userService.setLanguage(language);
|
||||
return await this.userService.updateUser(body, id, user.customer_id);
|
||||
|
||||
@@ -148,20 +148,23 @@ export class UsersService implements OnModuleInit {
|
||||
}
|
||||
|
||||
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
|
||||
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
|
||||
if (roleNames) {
|
||||
const { roleNames, ...updateUserDTO } = req;
|
||||
if (roleNames && roleNames.length > 0) {
|
||||
await this.setRoles({ roleNames, userId: id }, customerId);
|
||||
}
|
||||
|
||||
const { user } = await lastValueFrom(
|
||||
this.usersClientService.UserUpdate({
|
||||
name,
|
||||
department: updateUserDTO.department,
|
||||
email: updateUserDTO.email,
|
||||
hierarchy: updateUserDTO.hierarchy,
|
||||
jobTitle: updateUserDTO.jobTitle,
|
||||
name: updateUserDTO.name,
|
||||
bio: updateUserDTO.bio,
|
||||
companyName: updateUserDTO.companyName,
|
||||
companySite: updateUserDTO.companySite,
|
||||
customerId,
|
||||
id,
|
||||
department,
|
||||
hierarchy,
|
||||
jobTitle,
|
||||
email,
|
||||
metabaseUserId: undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user