gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-08-31 19:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

FEAT: trigger and get dataset cataloging task

Browse Source
This commit is contained in:
Gabriel Amorim
2023-03-14 13:39:21 -03:00
parent faf6b1df75
commit 2abc19c589
7 changed files with 265 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docsfera.external.json
+1 -1
View File
@@ -1544,7 +1544,7 @@
}
},
"info": {
"title": "Maestro - fix/delete-connector",
"title": "Maestro - main",
"description": "This is the Maestro API",
"version": "1.0.0",
"contact": {}
docsfera.json
+139 -1
View File
@@ -4190,6 +4190,118 @@
]
}
},
"/catalog/dataset-catalog-task": {
"post": {
"operationId": "CatalogController_triggerCatalog",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TriggerCatalogReq"
}
}
}
},
"responses": {
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TriggerCatalogRes"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/catalog/dataset-catalog-task/{session}": {
"get": {
"operationId": "CatalogController_getCatalogTask",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
},
{
"name": "session",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
},
"201": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TriggerCatalogRes"
}
}
}
}
},
"tags": [
"Catalog"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/terms-of-use/token": {
"get": {
"operationId": "TermsOfUseController_getToken",
@@ -4716,7 +4828,7 @@
}
},
"info": {
"title": "Maestro - fix/delete-connector",
"title": "Maestro - main",
"description": "This is the Maestro API",
"version": "1.0.0",
"contact": {}
@@ -7083,6 +7195,32 @@
"comment"
]
},
"TriggerCatalogReq": {
"type": "object",
"properties": {
"pipeline_id": {
"type": "string"
},
"table_name": {
"type": "string"
}
},
"required": [
"pipeline_id",
"table_name"
]
},
"TriggerCatalogRes": {
"type": "object",
"properties": {
"session": {
"type": "string"
}
},
"required": [
"session"
]
},
"ConnectionTestPingReq": {
"type": "object",
"properties": {
package-lock.json
Generated
+32 -23
View File
@@ -12,7 +12,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.29.1",
"@dadosfera/protospack-v2": "3.30.0-beta.2",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.29.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1.tgz",
"integrity": "sha512-uVswwXltdG0aX8cYVSFPRcROPWjOlW5bJGFfn5RNodKyHeBiTVDWBB3/YgocKrWV4mYBSbZ11ctBzjSAOkf/0Q==",
"version": "3.30.0-beta.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.30.0-beta.2.tgz",
"integrity": "sha512-/8LftDebU8Vno07jxSiIXF/D1cxXUuN+wjz+REXiAttRCUbEwK2/kAEpiLu+9gscjdA+Bj1aqmtLcVbvpMQE/w==",
"dependencies": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -4495,9 +4495,10 @@
"license": "MIT"
},
"node_modules/cookiejar": {
"version": "2.1.3",
"dev": true,
"license": "MIT"
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
"integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==",
"dev": true
},
"node_modules/core-util-is": {
"version": "1.0.3",
@@ -7759,9 +7760,10 @@
"license": "ISC"
},
"node_modules/json5": {
"version": "2.2.1",
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"dev": true,
"license": "MIT",
"bin": {
"json5": "lib/cli.js"
},
@@ -7968,9 +7970,9 @@
}
},
"node_modules/luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==",
"version": "1.28.1",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.1.tgz",
"integrity": "sha512-gYHAa180mKrNIUJCbwpmD0aTu9kV0dREDrwNnuyFAsO1Wt0EVYSZelPnJlbj9HplzXX/YWXHFTL45kvZ53M0pw==",
"engines": {
"node": "*"
}
@@ -10384,9 +10386,10 @@
}
},
"node_modules/tsconfig-paths/node_modules/json5": {
"version": "1.0.1",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
"integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"minimist": "^1.2.0"
},
@@ -12284,9 +12287,9 @@
}
},
"@dadosfera/protospack-v2": {
"version": "3.29.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1.tgz",
"integrity": "sha512-uVswwXltdG0aX8cYVSFPRcROPWjOlW5bJGFfn5RNodKyHeBiTVDWBB3/YgocKrWV4mYBSbZ11ctBzjSAOkf/0Q==",
"version": "3.30.0-beta.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.30.0-beta.2.tgz",
"integrity": "sha512-/8LftDebU8Vno07jxSiIXF/D1cxXUuN+wjz+REXiAttRCUbEwK2/kAEpiLu+9gscjdA+Bj1aqmtLcVbvpMQE/w==",
"requires": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -14190,7 +14193,9 @@
"version": "1.0.6"
},
"cookiejar": {
"version": "2.1.3",
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
"integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==",
"dev": true
},
"core-util-is": {
@@ -16399,7 +16404,9 @@
"dev": true
},
"json5": {
"version": "2.2.1",
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"dev": true
},
"jsonc-parser": {
@@ -16548,9 +16555,9 @@
}
},
"luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
"version": "1.28.1",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.1.tgz",
"integrity": "sha512-gYHAa180mKrNIUJCbwpmD0aTu9kV0dREDrwNnuyFAsO1Wt0EVYSZelPnJlbj9HplzXX/YWXHFTL45kvZ53M0pw=="
},
"macos-release": {
"version": "2.5.0",
@@ -18138,7 +18145,9 @@
},
"dependencies": {
"json5": {
"version": "1.0.1",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
"integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
"dev": true,
"requires": {
"minimist": "^1.2.0"
package.json
+1 -1
View File
@@ -29,7 +29,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.29.1",
"@dadosfera/protospack-v2": "3.30.0-beta.2",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
src/modules/catalog/catalog.controller.ts
+50 -1
View File
@@ -14,7 +14,7 @@ import {
Query,
UseFilters,
} from '@nestjs/common';
import { ApiHeaders, ApiTags } from '@nestjs/swagger';
import { ApiCreatedResponse, ApiHeaders, ApiTags } from '@nestjs/swagger';
import {
Authenticated,
RequireSomePermission,
@@ -35,6 +35,8 @@ import {
IOneDataAsset,
IPreviewResponse,
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
@@ -579,4 +581,51 @@ export class CatalogController {
return response;
}
@Post('dataset-catalog-task')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@ApiCreatedResponse({ type: TriggerCatalogRes })
async triggerCatalog(
@User() user: RequestUser,
@Body() body: TriggerCatalogReq,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const session = await this.catalogService.triggerCatalog(body, metadata);
return { session };
}
@Get('dataset-catalog-task/:session')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
@ApiCreatedResponse({ type: TriggerCatalogRes })
async getCatalogTask(
@User() user: RequestUser,
@Param('session') session: string,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.getDatasetCatalogTask(
session,
metadata,
);
return response;
}
}
src/modules/catalog/catalog.service.ts
+13 -1
View File
@@ -18,7 +18,7 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { IUpdateDataRequest } from './dtos';
import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -333,6 +333,18 @@ class CatalogService implements OnModuleInit {
} as typeof data_asset;
});
}
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
const { session } = await lastValueFrom(
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
);
return session;
}
async getDatasetCatalogTask(session: string, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
);
return res;
}
}
export { CatalogService };
src/modules/catalog/dtos/index.ts
+29
View File
@@ -271,3 +271,32 @@ export class IColumnsMetadataResponse {
@ApiProperty({ type: [IColumnMetadata] })
columns_metadata: IColumnMetadata[];
}
export class TriggerCatalogReq {
@ApiProperty()
pipeline_id: string;
@ApiProperty()
table_name: string;
}
export class TriggerCatalogRes {
@ApiProperty()
session: string;
}
export class GetDatasetCatalogTaskRes {
@ApiProperty()
created_at: string;
@ApiProperty()
customer_name: string;
@ApiProperty()
session_id: string;
@ApiProperty()
status: string;
@ApiProperty()
table_name: string;
@ApiProperty()
updated_at: string;
@ApiProperty()
updated_by: string;
}