gitea_admin/maestro
1
0
Fork 0
mirror of https://github.com/dadosfera/maestro.git synced 2026-08-31 19:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

FIX: fix catalog route to use openSearch

Browse Source
This commit is contained in:
rodrigo.zamboni
2022-09-19 11:14:23 -03:00
parent 3d39affc61
commit c3877657ef
8 changed files with 133 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+22 -20
View File
@@ -11,8 +11,8 @@
"license": "UNLICENSED",
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
"@dadosfera/protospack-v2": "3.17.5",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.17.5",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -1716,9 +1716,9 @@
}
},
"node_modules/@dadosfera/dadosfera-logs": {
"version": "1.0.0-beta.3",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/dadosfera-logs/-/dadosfera-logs-1.0.0-beta.3.tgz",
"integrity": "sha512-VuGiQlpDulcl1+FSFoT4/6PA9Pr+DAzbFHAYesz9rPinR46ejG7ZBZk3m86NYqQtgMyurqXvgCR8GPy9FpwqFw==",
"version": "1.0.0-beta.4",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/dadosfera-logs/-/dadosfera-logs-1.0.0-beta.4.tgz",
"integrity": "sha512-fFncbNVMbfQa2EFl7NJgmtuSRWHGtmwCbl5p++lSLvGcTIMjLGJ4HAMzSMemQhSxiiYlokdwweb7jJY7Ms62SQ==",
"dependencies": {
"@elastic/ecs-winston-format": "^1.3.1",
"winston": "^3.7.2",
@@ -7975,9 +7975,9 @@
}
},
"node_modules/logform": {
"version": "2.4.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/logform/-/logform-2.4.1.tgz",
"integrity": "sha512-7XB/tqc3VRbri9pRjU6E97mQ8vC27ivJ3lct4jhyT+n0JNDd4YKldFl0D75NqDp46hk8RC7Ma1Vjv/UPf67S+A==",
"version": "2.4.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/logform/-/logform-2.4.2.tgz",
"integrity": "sha512-W4c9himeAwXEdZ05dQNerhFz2XG80P9Oj0loPUMV23VC2it0orMHQhJm4hdnnor3rd1HsGf6a2lPwBM1zeXHGw==",
"dependencies": {
"@colors/colors": "1.5.0",
"fecha": "^4.2.0",
@@ -10856,10 +10856,11 @@
}
},
"node_modules/winston": {
"version": "3.7.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/winston/-/winston-3.7.2.tgz",
"integrity": "sha512-QziIqtojHBoyzUOdQvQiar1DH0Xp9nF1A1y7NVy2DGEsz82SBDtOalS0ulTRGVT14xPX3WRWkCsdcJKqNflKng==",
"version": "3.8.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/winston/-/winston-3.8.2.tgz",
"integrity": "sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==",
"dependencies": {
"@colors/colors": "1.5.0",
"@dabh/diagnostics": "^2.0.2",
"async": "^3.2.3",
"is-stream": "^2.0.0",
@@ -12295,9 +12296,9 @@
}
},
"@dadosfera/dadosfera-logs": {
"version": "1.0.0-beta.3",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/dadosfera-logs/-/dadosfera-logs-1.0.0-beta.3.tgz",
"integrity": "sha512-VuGiQlpDulcl1+FSFoT4/6PA9Pr+DAzbFHAYesz9rPinR46ejG7ZBZk3m86NYqQtgMyurqXvgCR8GPy9FpwqFw==",
"version": "1.0.0-beta.4",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/dadosfera-logs/-/dadosfera-logs-1.0.0-beta.4.tgz",
"integrity": "sha512-fFncbNVMbfQa2EFl7NJgmtuSRWHGtmwCbl5p++lSLvGcTIMjLGJ4HAMzSMemQhSxiiYlokdwweb7jJY7Ms62SQ==",
"requires": {
"@elastic/ecs-winston-format": "^1.3.1",
"winston": "^3.7.2",
@@ -16568,9 +16569,9 @@
}
},
"logform": {
"version": "2.4.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/logform/-/logform-2.4.1.tgz",
"integrity": "sha512-7XB/tqc3VRbri9pRjU6E97mQ8vC27ivJ3lct4jhyT+n0JNDd4YKldFl0D75NqDp46hk8RC7Ma1Vjv/UPf67S+A==",
"version": "2.4.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/logform/-/logform-2.4.2.tgz",
"integrity": "sha512-W4c9himeAwXEdZ05dQNerhFz2XG80P9Oj0loPUMV23VC2it0orMHQhJm4hdnnor3rd1HsGf6a2lPwBM1zeXHGw==",
"requires": {
"@colors/colors": "1.5.0",
"fecha": "^4.2.0",
@@ -18486,10 +18487,11 @@
}
},
"winston": {
"version": "3.7.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/winston/-/winston-3.7.2.tgz",
"integrity": "sha512-QziIqtojHBoyzUOdQvQiar1DH0Xp9nF1A1y7NVy2DGEsz82SBDtOalS0ulTRGVT14xPX3WRWkCsdcJKqNflKng==",
"version": "3.8.2",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/winston/-/winston-3.8.2.tgz",
"integrity": "sha512-MsE1gRx1m5jdTTO9Ld/vND4krP2To+lgDoMEHGGa4HIlAUyXJtfc7CxQcGXVyz2IBpw5hbFkj2b/AtUdQwyRew==",
"requires": {
"@colors/colors": "1.5.0",
"@dabh/diagnostics": "^2.0.2",
"async": "^3.2.3",
"is-stream": "^2.0.0",
package.json
+2 -2
View File
@@ -26,8 +26,8 @@
},
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.3",
"@dadosfera/protospack-v2": "3.17.5",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "^3.17.5",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
src/app.module.ts
+1
View File
@@ -53,6 +53,7 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
InputsModule,
OauthModule,
PipelinesV2Module,
CatalogModule,
],
})
export class AppModule {}
src/modules/catalog/catalog-client.ts
+35
View File
@@ -0,0 +1,35 @@
import {
ClientsProviderAsyncOptions,
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { credentials } from '@grpc/grpc-js';
import { Catalog } from '@dadosfera/protospack-v2';
export class CatalogClientConfiguration {
public name = 'CatalogClientConfiguration';
private config: GrpcOptions = {
transport: Transport.GRPC,
options: {
url: process.env.PIFACTORY_URL,
package: [
Catalog.ProtoPackages.ReadPackage,
Catalog.ProtoPackages.WritePackage,
],
protoPath: [
Catalog.ProtoPaths.ReadFilePath,
Catalog.ProtoPaths.WriteFilePath,
],
loader: {
keepCase: true,
enums: String,
defaults: false,
},
},
};
providerOptions: ClientsProviderAsyncOptions = {
name: this.name,
...this.config,
};
}
src/modules/catalog/catalog.controller.ts
+13 -2
View File
@@ -13,6 +13,7 @@ import { AuthenticateCondition } from '../../authentication/authentication.decor
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@ApiTags('Catalog')
@Controller('catalog')
@@ -48,14 +49,24 @@ export class CatalogController {
@Get('all')
async catalogAll(@Body() body) {
const { user_id, customer, customer_id } = body.info;
this.logger.info(`/catalog - ON CATALOG ALL ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const res = await this.catalogService.catalogAll(body);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name: customer,
});
return res;
const res = await this.catalogService.getAllOpenSearchDataAssets(
body,
metadata,
);
return { tables: res.data_assets };
}
@Get('data_apps')
src/modules/catalog/catalog.module.ts
+11
View File
@@ -4,9 +4,20 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { PipelinesService } from '../pipelinesV2/pipelines.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
OldPipelineModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
exports: [CatalogService],
})
export class CatalogModule {}
src/modules/catalog/catalog.service.ts
+48 -1
View File
@@ -1,6 +1,32 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { ReadService } from '@dadosfera/protospack-v2/dist/lib/Catalog';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Catalog';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/Catalog';
import { Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import axios from 'axios';
import { lastValueFrom } from 'rxjs';
import { CatalogClientConfiguration } from './catalog-client';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: any;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
class CatalogService {
_getNimbusUrl(body) {
const customer = body.info.customer.toLowerCase();
@@ -41,6 +67,27 @@ class CatalogService {
return data;
}
async getAllOpenSearchDataAssets(
data: Messages.GetAllDataAssetsRequest,
metadata,
) {
this.logger.info('CatalogService - getAllOpenSearchDashboard');
const response = await lastValueFrom(
this.catalogReadService.GetAllDataAssets(data, metadata),
)
.then((res) => {
this.logger.info('Done');
return JSON.parse(res.data_assets);
})
.catch((err) => {
this.logger.error(err.message);
throw new Error(err);
});
return { data_assets: response };
}
async getOneDashboardMetabase(id, body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.get(
swagger.txt
+1 -1
View File
File diff suppressed because one or more lines are too long