mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-18 21:59:08 +00:00
Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
003c9b176c | ||
|
|
1e5fe59d9c | ||
|
|
1a2b9617fe | ||
|
|
1f92510a43 | ||
|
|
f7fceb1ea2 | ||
|
|
9bc42a4af0 | ||
|
|
5c98cc1748 | ||
|
|
9ae14ab04a | ||
|
|
5d014f4669 | ||
|
|
5e8dea7255 | ||
|
|
38fc9f521b | ||
|
|
80d7ac6ff9 | ||
|
|
9bacd7a0f5 | ||
|
|
c89ac40039 | ||
|
|
6d6cf590ee | ||
|
|
f063a8f30e | ||
|
|
03536a2180 | ||
|
|
0efd5cc257 | ||
|
|
9439f13b7b | ||
|
|
d936806406 | ||
|
|
9d7ccd4201 | ||
|
|
c8a4c1aae6 | ||
|
|
406db7ccff | ||
|
|
f48a35c7ce | ||
|
|
8d25df74b7 | ||
|
|
831a03402e | ||
|
|
a631338599 | ||
|
|
d943721b4a | ||
|
|
1bbf978cc4 | ||
|
|
856f913ce6 | ||
|
|
dc617b6925 | ||
|
|
3f80062672 | ||
|
|
07eb423814 | ||
|
|
721232448e | ||
|
|
0b0d261aae | ||
|
|
87a8d741d7 |
@@ -4787,6 +4787,41 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/catalog/register-dataset": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "CatalogController_registerDatasetWithMetadataRequest",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"name": "dadosfera-lang",
|
||||||
|
"in": "header",
|
||||||
|
"required": false,
|
||||||
|
"schema": {
|
||||||
|
"enum": [
|
||||||
|
"pt-br",
|
||||||
|
"en-us"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"201": {
|
||||||
|
"description": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tags": [
|
||||||
|
"Catalog"
|
||||||
|
],
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"access-token": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"access-token": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/terms-of-use/token": {
|
"/terms-of-use/token": {
|
||||||
"get": {
|
"get": {
|
||||||
"operationId": "TermsOfUseController_getToken",
|
"operationId": "TermsOfUseController_getToken",
|
||||||
@@ -5454,6 +5489,14 @@
|
|||||||
"schema": {
|
"schema": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "origin",
|
||||||
|
"required": true,
|
||||||
|
"in": "header",
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"responses": {
|
"responses": {
|
||||||
@@ -5608,6 +5651,9 @@
|
|||||||
"name": {
|
"name": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"displayName": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"tier": {
|
"tier": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
@@ -5628,6 +5674,7 @@
|
|||||||
"modules",
|
"modules",
|
||||||
"id",
|
"id",
|
||||||
"name",
|
"name",
|
||||||
|
"displayName",
|
||||||
"tier",
|
"tier",
|
||||||
"scheduleLimit",
|
"scheduleLimit",
|
||||||
"links",
|
"links",
|
||||||
|
|||||||
+3
-1
@@ -17,4 +17,6 @@ charts:
|
|||||||
- name: maestro.open_customer_id
|
- name: maestro.open_customer_id
|
||||||
value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
|
value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
|
||||||
- name: maestro.open_group_id
|
- name: maestro.open_group_id
|
||||||
value: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
value: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
||||||
|
- name: replicaCount
|
||||||
|
value: 2
|
||||||
|
|||||||
+3
-3
@@ -9,11 +9,11 @@ charts:
|
|||||||
- name: hostname
|
- name: hostname
|
||||||
value: maestro.stg.dadosfera.ai
|
value: maestro.stg.dadosfera.ai
|
||||||
- name: maestro.pi_factory_url
|
- name: maestro.pi_factory_url
|
||||||
value: pi-factory.dadosfera.ai
|
value: pi-factory.stg.dadosfera.ai
|
||||||
- name: maestro.in_factory_url
|
- name: maestro.in_factory_url
|
||||||
value: in-factory.dadosfera.ai
|
value: in-factory.stg.dadosfera.ai
|
||||||
- name: maestro.tr_factory_url
|
- name: maestro.tr_factory_url
|
||||||
value: in-factory.dadosfera.ai
|
value: in-factory.stg.dadosfera.ai
|
||||||
- name: maestro.open_customer_id
|
- name: maestro.open_customer_id
|
||||||
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||||
- name: maestro.open_group_id
|
- name: maestro.open_group_id
|
||||||
|
|||||||
@@ -35,7 +35,8 @@ spec:
|
|||||||
- key: application
|
- key: application
|
||||||
operator: In
|
operator: In
|
||||||
values:
|
values:
|
||||||
- backend
|
- general
|
||||||
|
|
||||||
tolerations:
|
tolerations:
|
||||||
- key: "kubernetes.azure.com/scalesetpriority"
|
- key: "kubernetes.azure.com/scalesetpriority"
|
||||||
operator: "Equal"
|
operator: "Equal"
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
apiVersion: networking.k8s.io/v1
|
||||||
|
kind: Ingress
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
|
||||||
|
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||||
|
nginx.ingress.kubernetes.io/server-snippet: |
|
||||||
|
underscores_in_headers on;
|
||||||
|
ignore_invalid_headers on;
|
||||||
|
|
||||||
|
generation: 1
|
||||||
|
labels:
|
||||||
|
app: maestro
|
||||||
|
name: open-data
|
||||||
|
namespace: applications
|
||||||
|
spec:
|
||||||
|
ingressClassName: nginx
|
||||||
|
rules:
|
||||||
|
- host: {{ .Values.hostname }}
|
||||||
|
http:
|
||||||
|
paths:
|
||||||
|
- backend:
|
||||||
|
service:
|
||||||
|
name: maestro
|
||||||
|
port:
|
||||||
|
number: {{ .Values.ingress.port }}
|
||||||
|
path: /open-data/sharing-ocean-data
|
||||||
|
pathType: Prefix
|
||||||
+1
-1
@@ -3,7 +3,7 @@
|
|||||||
# Declare variables to be passed into your templates.
|
# Declare variables to be passed into your templates.
|
||||||
|
|
||||||
replicaCount: 3
|
replicaCount: 3
|
||||||
hostname: maestro-temp.dadosfera.ai
|
hostname: maestro.dadosfera.ai
|
||||||
image:
|
image:
|
||||||
repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
|
repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
|
||||||
pullPolicy: IfNotPresent
|
pullPolicy: IfNotPresent
|
||||||
|
|||||||
Generated
+1617
-1401
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -30,7 +30,7 @@
|
|||||||
"@aws-sdk/client-secrets-manager": "^3.414.0",
|
"@aws-sdk/client-secrets-manager": "^3.414.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack": "2.5.3",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
"@dadosfera/protospack-v2": "3.37.0-beta.5",
|
"@dadosfera/protospack-v2": "^3.37.0-beta.10 ",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
|
|||||||
@@ -72,6 +72,8 @@ export class AuthCustomer {
|
|||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
name: string;
|
name: string;
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
|
displayName: string;
|
||||||
|
@ApiProperty()
|
||||||
tier: string;
|
tier: string;
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
scheduleLimit: string;
|
scheduleLimit: string;
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
} from '@nestjs/microservices';
|
} from '@nestjs/microservices';
|
||||||
import { credentials } from '@grpc/grpc-js';
|
import { credentials } from '@grpc/grpc-js';
|
||||||
import { Catalog } from '@dadosfera/protospack-v2';
|
import { Catalog } from '@dadosfera/protospack-v2';
|
||||||
|
import { PlatformInterfaces } from '@dadosfera/protospack-v2';
|
||||||
|
|
||||||
const isLocalConnection =
|
const isLocalConnection =
|
||||||
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
|
||||||
@@ -19,11 +20,13 @@ export class CatalogClientConfiguration {
|
|||||||
package: [
|
package: [
|
||||||
Catalog.ProtoPackages.ReadPackage,
|
Catalog.ProtoPackages.ReadPackage,
|
||||||
Catalog.ProtoPackages.WritePackage,
|
Catalog.ProtoPackages.WritePackage,
|
||||||
|
PlatformInterfaces.ProtoPackages.WritePackage
|
||||||
],
|
],
|
||||||
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
credentials: isLocalConnection ? undefined : credentials.createSsl(),
|
||||||
protoPath: [
|
protoPath: [
|
||||||
Catalog.ProtoPaths.ReadFilePath,
|
Catalog.ProtoPaths.ReadFilePath,
|
||||||
Catalog.ProtoPaths.WriteFilePath,
|
Catalog.ProtoPaths.WriteFilePath,
|
||||||
|
PlatformInterfaces.ProtoPaths.WriteFilePath
|
||||||
],
|
],
|
||||||
loader: {
|
loader: {
|
||||||
keepCase: true,
|
keepCase: true,
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ import {
|
|||||||
Put,
|
Put,
|
||||||
Query,
|
Query,
|
||||||
UseFilters,
|
UseFilters,
|
||||||
|
HttpException,
|
||||||
|
HttpStatus,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import {
|
import {
|
||||||
ApiCreatedResponse,
|
ApiCreatedResponse,
|
||||||
@@ -30,6 +32,7 @@ import { CatalogService } from './catalog.service';
|
|||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||||
|
import { lastValueFrom } from 'rxjs';
|
||||||
import {
|
import {
|
||||||
BatchRemoveRlsRulesRequest,
|
BatchRemoveRlsRulesRequest,
|
||||||
GetDatasetCatalogTaskRes,
|
GetDatasetCatalogTaskRes,
|
||||||
@@ -54,6 +57,7 @@ import {
|
|||||||
AddRlsRuleRequest,
|
AddRlsRuleRequest,
|
||||||
GetNimbusDashboardsRequest,
|
GetNimbusDashboardsRequest,
|
||||||
GetRlsRulesRequest,
|
GetRlsRulesRequest,
|
||||||
|
RegisterDatasetWithMetatadaRequest,
|
||||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||||
|
|
||||||
@ApiTags('Catalog')
|
@ApiTags('Catalog')
|
||||||
@@ -743,4 +747,94 @@ export class CatalogController {
|
|||||||
|
|
||||||
return JSON.parse(dashboards);
|
return JSON.parse(dashboards);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@Post('register-dataset')
|
||||||
|
@RequireSomePermission(
|
||||||
|
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
||||||
|
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
|
||||||
|
)
|
||||||
|
async registerDatasetWithMetadataRequest(
|
||||||
|
@Body() body: RegisterDatasetWithMetatadaRequest,
|
||||||
|
@User() user: RequestUser,
|
||||||
|
) {
|
||||||
|
const { customer_id, customer_name, user_id, username } = user;
|
||||||
|
const logMetadata = {
|
||||||
|
customer_name: customer_name,
|
||||||
|
user_id: user_id,
|
||||||
|
method: 'POST',
|
||||||
|
path: '/catalog/register-dataset',
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
|
||||||
|
const metadata = PackTheMetadata({
|
||||||
|
customer_id,
|
||||||
|
customer_name,
|
||||||
|
user_id,
|
||||||
|
username,
|
||||||
|
});
|
||||||
|
|
||||||
|
this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
|
||||||
|
|
||||||
|
// Create table metadata
|
||||||
|
const tableMetadataBody = {
|
||||||
|
table_metadata: body.table_metadata,
|
||||||
|
info: {
|
||||||
|
customer: customer_name,
|
||||||
|
|
||||||
|
},
|
||||||
|
logMetadata: logMetadata
|
||||||
|
};
|
||||||
|
|
||||||
|
const table_metadata_id = await this.catalogService.createTableMetadata(
|
||||||
|
tableMetadataBody,
|
||||||
|
);
|
||||||
|
this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
|
||||||
|
|
||||||
|
// Create column metadata
|
||||||
|
const columnMetadataBody = {
|
||||||
|
column_metadata: body.column_metadata,
|
||||||
|
info: {
|
||||||
|
customer: customer_name,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
|
||||||
|
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
|
||||||
|
|
||||||
|
// Create data preview
|
||||||
|
const dataPreviewBody = {
|
||||||
|
data_preview: body.data_preview,
|
||||||
|
info: {
|
||||||
|
customer: customer_name,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
|
||||||
|
const data_preview_id = await this.catalogService.createDataPreview(
|
||||||
|
dataPreviewBody,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Catalog dataset item
|
||||||
|
this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
|
||||||
|
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
|
||||||
|
this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
|
||||||
|
return {
|
||||||
|
message: 'Dataset registered successfully',
|
||||||
|
table_metadata_id: table_metadata_id,
|
||||||
|
column_metadata_ids: column_metadata_ids,
|
||||||
|
data_preview_id: data_preview_id,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
|
||||||
|
|
||||||
|
throw new HttpException(
|
||||||
|
{
|
||||||
|
message: 'Ocorreu um erro ao registrar o dataset',
|
||||||
|
error: error.message,
|
||||||
|
code: 'REGISTRATION_FAILED',
|
||||||
|
details: error.message,
|
||||||
|
},
|
||||||
|
HttpStatus.INTERNAL_SERVER_ERROR,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,11 @@ import {
|
|||||||
ProtoServices,
|
ProtoServices,
|
||||||
Messages,
|
Messages,
|
||||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
|
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
|
||||||
|
import {
|
||||||
|
Messages as PlatformInterfaceMessages,
|
||||||
|
WriteService as PlatformInterfaceWriteService,
|
||||||
|
ProtoServices as PlatformInterfacesProtoServices,
|
||||||
|
} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
|
||||||
import {
|
import {
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
HttpException,
|
HttpException,
|
||||||
@@ -33,6 +38,7 @@ import {
|
|||||||
class CatalogService implements OnModuleInit {
|
class CatalogService implements OnModuleInit {
|
||||||
catalogReadService: ReadService.CatalogReadServices;
|
catalogReadService: ReadService.CatalogReadServices;
|
||||||
catalogWriteService: WriteService.CatalogWriteServices;
|
catalogWriteService: WriteService.CatalogWriteServices;
|
||||||
|
platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
|
||||||
logger: any;
|
logger: any;
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DadosferaLogger)
|
@Inject(DadosferaLogger)
|
||||||
@@ -54,9 +60,14 @@ class CatalogService implements OnModuleInit {
|
|||||||
this.grpcClient.getService<WriteService.CatalogWriteServices>(
|
this.grpcClient.getService<WriteService.CatalogWriteServices>(
|
||||||
ProtoServices.CatalogWriteServices,
|
ProtoServices.CatalogWriteServices,
|
||||||
);
|
);
|
||||||
|
this.platformWriteService =
|
||||||
|
this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
|
||||||
|
PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
_getNimbusUrl(body) {
|
_getNimbusUrl(body) {
|
||||||
|
this.logger.debug(`Body: ${JSON.stringify(body)}`);
|
||||||
const customer = body.info.customer.toLowerCase();
|
const customer = body.info.customer.toLowerCase();
|
||||||
|
|
||||||
if (process.env.ENV === 'prd') {
|
if (process.env.ENV === 'prd') {
|
||||||
@@ -420,6 +431,112 @@ class CatalogService implements OnModuleInit {
|
|||||||
);
|
);
|
||||||
return res.dashboards;
|
return res.dashboards;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async createTableMetadata(body: any): Promise<number> {
|
||||||
|
const nimbusUrl = this._getNimbusUrl(body);
|
||||||
|
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||||
|
|
||||||
|
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
|
||||||
|
|
||||||
|
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
|
||||||
|
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||||
|
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
|
||||||
|
{...body.logMetadata},
|
||||||
|
);
|
||||||
|
return data.id;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
|
||||||
|
error.response?.status
|
||||||
|
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
|
||||||
|
throw new Error(error.response?.data?.message || error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async createColumnMetadata(body: any): Promise<number[]> {
|
||||||
|
const nimbusUrl = this._getNimbusUrl(body);
|
||||||
|
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
|
||||||
|
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
try {
|
||||||
|
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
|
||||||
|
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||||
|
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
|
||||||
|
const { data, status } = await axios.post(endpoint, body.column_metadata);
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
|
||||||
|
{...body.logMetadata},
|
||||||
|
);
|
||||||
|
return data.map((column) => column.id);
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
|
||||||
|
error.response?.status
|
||||||
|
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
|
||||||
|
throw new Error(error.response?.data?.message || error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async createDataPreview(body: any): Promise<number> {
|
||||||
|
const nimbusUrl = this._getNimbusUrl(body);
|
||||||
|
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||||
|
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
|
||||||
|
|
||||||
|
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
|
||||||
|
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||||
|
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { data, status } = await axios.post(endpoint, body.data_preview);
|
||||||
|
|
||||||
|
this.logger.info(
|
||||||
|
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
|
||||||
|
{...body.logMetadata},
|
||||||
|
);
|
||||||
|
return data.id;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
|
||||||
|
error.response?.status
|
||||||
|
} because of ${error.response?.data || error.message}`,
|
||||||
|
{...body.logMetadata},
|
||||||
|
);
|
||||||
|
throw new Error(error.response?.data?.message || error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
|
||||||
|
const customer_name_raw = metadata.get('customer_name');
|
||||||
|
|
||||||
|
const customer_name = customer_name_raw?.[0]?.toString();
|
||||||
|
if (!customer_name) {
|
||||||
|
throw new BadRequestException('Customer name not found in metadata');
|
||||||
|
}
|
||||||
|
const res = await lastValueFrom(
|
||||||
|
this.platformWriteService.CatalogDataAssets(
|
||||||
|
{
|
||||||
|
data_assets: [
|
||||||
|
{
|
||||||
|
data_asset_id: table_metadata_id.toString(),
|
||||||
|
customer_name: customer_name,
|
||||||
|
data_asset_type: 'dataset',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
metadata,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export { CatalogService };
|
export { CatalogService };
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
|
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
|
||||||
import { init } from 'mixpanel';
|
import { init } from 'mixpanel';
|
||||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||||
@@ -13,7 +13,12 @@ export class MixpanelController {
|
|||||||
private readonly mixpanelToken: string,
|
private readonly mixpanelToken: string,
|
||||||
) {}
|
) {}
|
||||||
@Post(':id')
|
@Post(':id')
|
||||||
async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
|
async trackEvent(
|
||||||
|
@Param('id') id,
|
||||||
|
@Body() body,
|
||||||
|
@User() user: RequestUser,
|
||||||
|
@Req() request
|
||||||
|
) {
|
||||||
delete body.info;
|
delete body.info;
|
||||||
const mixpanel = init(this.mixpanelToken);
|
const mixpanel = init(this.mixpanelToken);
|
||||||
|
|
||||||
@@ -66,13 +71,16 @@ export class MixpanelController {
|
|||||||
$email: user.username.includes('@')
|
$email: user.username.includes('@')
|
||||||
? user.username
|
? user.username
|
||||||
: user.username + '@dadosfera.ai',
|
: user.username + '@dadosfera.ai',
|
||||||
customer_name: user.customer_name,
|
customer_name: user.customer_name
|
||||||
});
|
});
|
||||||
|
|
||||||
await mixpanel.track(id, {
|
await mixpanel.track(id, {
|
||||||
distinct_id: user.username,
|
distinct_id: user.username,
|
||||||
customer: user.customer_name,
|
customer: user.customer_name,
|
||||||
env: process.env.ENV,
|
env: process.env.ENV,
|
||||||
|
$ip: request.ip,
|
||||||
|
$os: request.headers['sec-ch-ua-platform'] || '',
|
||||||
|
$browser: request.headers['user-agent'],
|
||||||
...body,
|
...body,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||||
import { Body, Controller, ForbiddenException, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
|
import { Body, Controller, ForbiddenException, Header, Headers, HttpCode, HttpException, Inject, Param, Post, Query, Req, Res, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
|
||||||
import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
|
import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
|
||||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||||
@@ -40,20 +40,23 @@ export class OpenDataController {
|
|||||||
language: string,
|
language: string,
|
||||||
@Req()
|
@Req()
|
||||||
request: Request,
|
request: Request,
|
||||||
|
@Headers('origin')
|
||||||
|
origin: string
|
||||||
) {
|
) {
|
||||||
if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) {
|
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
|
||||||
|
|
||||||
|
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
|
||||||
|
if (!corslist.includes(origin)) {
|
||||||
|
this.logger.info('block request by cors list: '+ origin);
|
||||||
throw new ForbiddenException();
|
throw new ForbiddenException();
|
||||||
}
|
}
|
||||||
this.logger.info('createUser for open data');
|
|
||||||
|
|
||||||
this.logger.info('user-agent', request.headers['user-agent']);
|
|
||||||
|
|
||||||
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
|
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
|
||||||
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
|
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
|
||||||
const roles = [process.env.OPEN_GROUP_ID];
|
const roles = [process.env.OPEN_GROUP_ID];
|
||||||
const metadata = PackTheMetadata({
|
const metadata = PackTheMetadata({
|
||||||
language: language || 'en-us'
|
language: language || 'en-us'
|
||||||
});
|
});
|
||||||
|
|
||||||
const data = {}
|
const data = {}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user