gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-08-31 19:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #487 from dadosfera/feature/table-schema-filter

Browse Source 
FEAT: list assets by pipeline and multiple ids
This commit is contained in:
Marcos Rodrigues Silva
2026-04-29 16:50:52 -03:00
committed by GitHub
parent cf94c73648 02627d023a
commit f449d8ebd9
8 changed files with 151 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docsfera.json
+67 -14
View File
@@ -3541,6 +3541,61 @@
]
}
},
"/pipelinesV2/{id}/data-assets": {
"get": {
"operationId": "PipelinesController_findAllDataAssetByPipeline",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
},
{
"name": "object",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "string"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
}
]
}
},
"/pipelinesV2/{pipelineId}/inputs/{id}": {
"patch": {
"operationId": "PipelinesController_updatePipelineInput",
@@ -4001,7 +4056,7 @@
]
}
},
"/platform/pipeline": {
"/platform/pipelines": {
"post": {
"operationId": "PlatformApiController_createPipeline",
"summary": "Create a new pipeline",
@@ -4026,9 +4081,7 @@
"access-token": []
}
]
}
},
"/platform/pipelines": {
},
"get": {
"operationId": "PlatformApiController_getPipelines",
"summary": "List all pipelines for customer",
@@ -4055,7 +4108,7 @@
]
}
},
"/platform/pipeline/{pipelineId}": {
"/platform/pipelines/{pipelineId}": {
"get": {
"operationId": "PlatformApiController_getPipeline",
"summary": "Get pipeline by ID",
@@ -4159,7 +4212,7 @@
]
}
},
"/platform/pipeline/execute": {
"/platform/pipelines/execute": {
"post": {
"operationId": "PlatformApiController_executePipeline",
"summary": "Execute a pipeline",
@@ -4186,7 +4239,7 @@
]
}
},
"/platform/pipeline/pause": {
"/platform/pipelines/pause": {
"post": {
"operationId": "PlatformApiController_pausePipeline",
"summary": "Pause a pipeline",
@@ -4213,7 +4266,7 @@
]
}
},
"/platform/pipeline/unpause": {
"/platform/pipelines/unpause": {
"post": {
"operationId": "PlatformApiController_unpausePipeline",
"summary": "Unpause a pipeline",
@@ -4240,7 +4293,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/memory": {
"/platform/pipelines/{pipelineId}/memory": {
"put": {
"operationId": "PlatformApiController_updatePipelineMemory",
"summary": "Update pipeline memory configuration",
@@ -4276,7 +4329,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/metadata": {
"/platform/pipelines/{pipelineId}/metadata": {
"put": {
"operationId": "PlatformApiController_updatePipelineMetadata",
"summary": "Update pipeline metadata",
@@ -4403,7 +4456,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run": {
"/platform/pipelines/{pipelineId}/pipeline_run": {
"get": {
"operationId": "PlatformApiController_getPipelineRuns",
"summary": "Get pipeline runs for a pipeline",
@@ -4439,7 +4492,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run/{runId}": {
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}": {
"get": {
"operationId": "PlatformApiController_getPipelineRun",
"summary": "Get specific pipeline run",
@@ -4483,7 +4536,7 @@
]
}
},
"/platform/pipeline/pipeline_run/{runId}/logs": {
"/platform/pipelines/pipeline_run/{runId}/logs": {
"get": {
"operationId": "PlatformApiController_getPipelineRunLogs",
"summary": "Get pipeline run logs",
@@ -4519,7 +4572,7 @@
]
}
},
"/platform/pipeline/{pipelineId}/pipeline_run/{runId}/cancel": {
"/platform/pipelines/{pipelineId}/pipeline_run/{runId}/cancel": {
"post": {
"operationId": "PlatformApiController_cancelPipelineRun",
"summary": "Cancel a running pipeline run",
package-lock.json
Generated
+5 -4
View File
@@ -17,7 +17,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.7",
"@dadosfera/protospack-v2": "3.40.0-beta.8",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -1745,9 +1745,10 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.40.0-beta.7",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.7.tgz",
"integrity": "sha512-Ab9k+hkdigFOqIRafe7e+Cco/E/Z1DJmZnfqAV5DHcnE2okhApj9MuBDCHRChGnEh+cVAkOT2/mOx97aWdn5Ow==",
"version": "3.40.0-beta.8",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.8.tgz",
"integrity": "sha512-JE5qMjqB3UOM+tCUxB1EwYLQW0PecsaQIa1KDpKEaG3lrzG/H13z8iJi3WH/DuVav2EI94i9VcJWJ1Y0F7ribw==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
"rxjs": "^7.5.5"
package.json
+1 -1
View File
@@ -35,7 +35,7 @@
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.40.0-beta.7",
"@dadosfera/protospack-v2": "3.40.0-beta.8",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
src/authentication/authentication.guard.ts
+1
View File
@@ -153,6 +153,7 @@ export class AuthenticationGuard
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
roles: accessTokenPayload.roles,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
src/decorators/user.decorator.ts
+1
View File
@@ -12,6 +12,7 @@ export interface RequestUser {
customer_tier: string;
access_token: string;
customer_modules: string[];
roles: string[];
}
export const User: (options?: { required?: boolean }) => ParameterDecorator =
src/modules/pipelinesV2/pipelines.controller.ts
+26
View File
@@ -193,6 +193,7 @@ export class PipelinesController {
@Get(':id/status')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
@@ -247,6 +248,31 @@ export class PipelinesController {
return pipelineRes;
}
@Get("/:id/data-assets")
@RequireSomePermission(
PERMISSIONS_GROUPS.PIPELINE.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
PERMISSIONS_GROUPS.CATALOG.permissions.GET
)
async findAllDataAssetByPipeline(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser,
@Query('object') object: string
) {
const payload = {
pipeline: id,
object: object,
};
this.logger.info(`GET pipelinesV2/:id/data-assets` + JSON.stringify(payload));
const result =
await this.pipelinesClientService.findAllDataAssetByPipeline(payload, user);
return result;
}
@Patch('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async update(
src/modules/pipelinesV2/pipelines.module.ts
+3 -1
View File
@@ -14,6 +14,7 @@ import { TransformationsModule } from '../transformations/transformations.module
import { PlatformApiModule } from '../platform-api/platform-api.module';
import { NimbusServicesModule } from 'src/services/nimbus/nimbus.module';
import { NimbusService } from 'src/services/nimbus/nimbus.service';
import { CatalogModule } from '../catalog/catalog.module';
const client = new PipelinesClientConfiguration();
@@ -25,7 +26,8 @@ const client = new PipelinesClientConfiguration();
InputsModule,
TransformationsModule,
PlatformApiModule,
NimbusServicesModule
NimbusServicesModule,
CatalogModule
],
controllers: [PipelinesController],
providers: [PipelinesService, DadosferaLogger, NimbusService],
src/modules/pipelinesV2/pipelines.service.ts
+47 -1
View File
@@ -32,6 +32,10 @@ import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entitie
import { TableUpdate } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
import { AxiosError } from 'axios';
import { NimbusService } from 'src/services/nimbus/nimbus.service';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { IDataAsset } from '../catalog/dtos';
import { CatalogService } from '../catalog/catalog.service';
type RollbackPromise = () => Promise<any>;
@@ -48,7 +52,8 @@ export class PipelinesService implements OnModuleInit {
private readonly inputsService: InputsService,
private readonly transformationsService: TransformationsService,
private readonly platformAPI: PlatformApiService,
private readonly nimbusService: NimbusService
private readonly nimbusService: NimbusService,
private readonly catalogService: CatalogService
) {
this.logger = dadosferaLogger.logger;
}
@@ -577,4 +582,45 @@ export class PipelinesService implements OnModuleInit {
this.logger.info('Platform api response: ' + JSON.stringify(response));
}
async findAllDataAssetByPipeline(data: {
pipeline: string,
object?: string
}, user: RequestUser) {
const metadata = PackTheMetadata(user);
const isDataAdmin = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
let has_permission = false;
const {
data_assets: resultString
} = await lastValueFrom(
this.pipelineReadService.FindAllDataAssetByPipeline(data, metadata)
);
const result = JSON.parse(resultString) as any;
const data_assets: IDataAsset[] = []
result.forEach(data_asset => {
if (data_asset?.owner === user.username) has_permission = true;
for (const role of user.roles) {
if (data_asset.roles.includes(role)) has_permission = true;
}
if (data_asset.users.includes(user.user_id)) has_permission = true;
if (isDataAdmin || has_permission) {
delete data_asset.p_roles;
delete data_asset.p_users;
data_assets.push(data_asset as IDataAsset);
}
});
const assets = await this.catalogService.getAssetsUsersAndRoles(data_assets, user.customer_id);
return assets;
}
}