Compare commits

...
15 Commits
10 changed files with 358 additions and 833 deletions
+1 -1
View File
@@ -290,7 +290,7 @@ Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
### Making a Pull Request
1. Commit your changes
2. Open the Pull Request on GitHub
3. Send Pull Request link in microsfera Google Chat Group for review and possible approval
3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
## 🛠️ Built with
Some technologies used in this project:
+249 -756
View File
File diff suppressed because it is too large Load Diff
+54 -61
View File
@@ -12,7 +12,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.21.0",
"@dadosfera/protospack-v2": "3.22.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -36,7 +36,7 @@
"mixpanel": "^0.17.0",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.1.5",
"passport-forcedotcom": "^0.2.0",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
@@ -1727,9 +1727,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"version": "3.22.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
"integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
"dependencies": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -3699,8 +3699,9 @@
},
"node_modules/asap": {
"version": "2.0.6",
"dev": true,
"license": "MIT"
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
"dev": true
},
"node_modules/asn1.js": {
"version": "5.4.1",
@@ -4705,9 +4706,10 @@
}
},
"node_modules/dezalgo": {
"version": "1.0.3",
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
"integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
"dev": true,
"license": "ISC",
"dependencies": {
"asap": "^2.0.0",
"wrappy": "1"
@@ -5902,23 +5904,28 @@
}
},
"node_modules/formidable": {
"version": "2.0.1",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.1.tgz",
"integrity": "sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"dezalgo": "1.0.3",
"hexoid": "1.0.0",
"once": "1.4.0",
"qs": "6.9.3"
"dezalgo": "^1.0.4",
"hexoid": "^1.0.0",
"once": "^1.4.0",
"qs": "^6.11.0"
},
"funding": {
"url": "https://ko-fi.com/tunnckoCore/commissions"
}
},
"node_modules/formidable/node_modules/qs": {
"version": "6.9.3",
"version": "6.11.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
"integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
"dev": true,
"license": "BSD-3-Clause",
"dependencies": {
"side-channel": "^1.0.4"
},
"engines": {
"node": ">=0.6"
},
@@ -8713,24 +8720,11 @@
}
},
"node_modules/passport-forcedotcom": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.1.5.tgz",
"integrity": "sha512-raDRQqBJtTEHDxLkxLG+9sYE0xpPy9PqriMPbMy9jmpIcU2+mOq9PgczvUPrU8UjfcZp3gqkDHqujcUWiGOVNQ==",
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.2.0.tgz",
"integrity": "sha512-gRI5+zQK9iVAZ1hJgt3TkWLeDw8SYrNWUsmVk0d0RyvUviQ4mT52x3JbYRiyZefoeID8afmLsJH8VOHjWdPf3Q==",
"dependencies": {
"passport-oauth2": "1.3.x"
},
"engines": {
"node": ">= 0.4.0"
}
},
"node_modules/passport-forcedotcom/node_modules/passport-oauth2": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/passport-oauth2/-/passport-oauth2-1.3.0.tgz",
"integrity": "sha512-AisRXcC7Vx8WSx5C9O8UNib/Y91/IOphs5tImlOMs7T+QTEJdmS+OxgXSskbnDdGQT/8mU5T3yMT6136rBjYXg==",
"dependencies": {
"oauth": "0.9.x",
"passport-strategy": "1.x.x",
"uid2": "0.0.x"
"passport-oauth2": "^1.6.1"
},
"engines": {
"node": ">= 0.4.0"
@@ -12330,9 +12324,9 @@
}
},
"@dadosfera/protospack-v2": {
"version": "3.21.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
"integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
"version": "3.22.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.22.0.tgz",
"integrity": "sha512-tkHqNiSeuiGtuFieOOaTTaB1OOKzcFd+7bTlDXj6FdorIHXt+FtHxslWX5b9XERzkQg7nseak1gFw2w4krOdbg==",
"requires": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -13675,6 +13669,8 @@
},
"asap": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==",
"dev": true
},
"asn1.js": {
@@ -14372,7 +14368,9 @@
"dev": true
},
"dezalgo": {
"version": "1.0.3",
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
"integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
"dev": true,
"requires": {
"asap": "^2.0.0",
@@ -15214,18 +15212,25 @@
}
},
"formidable": {
"version": "2.0.1",
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.1.tgz",
"integrity": "sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==",
"dev": true,
"requires": {
"dezalgo": "1.0.3",
"hexoid": "1.0.0",
"once": "1.4.0",
"qs": "6.9.3"
"dezalgo": "^1.0.4",
"hexoid": "^1.0.0",
"once": "^1.4.0",
"qs": "^6.11.0"
},
"dependencies": {
"qs": {
"version": "6.9.3",
"dev": true
"version": "6.11.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
"integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
"dev": true,
"requires": {
"side-channel": "^1.0.4"
}
}
}
},
@@ -17093,23 +17098,11 @@
}
},
"passport-forcedotcom": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.1.5.tgz",
"integrity": "sha512-raDRQqBJtTEHDxLkxLG+9sYE0xpPy9PqriMPbMy9jmpIcU2+mOq9PgczvUPrU8UjfcZp3gqkDHqujcUWiGOVNQ==",
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/passport-forcedotcom/-/passport-forcedotcom-0.2.0.tgz",
"integrity": "sha512-gRI5+zQK9iVAZ1hJgt3TkWLeDw8SYrNWUsmVk0d0RyvUviQ4mT52x3JbYRiyZefoeID8afmLsJH8VOHjWdPf3Q==",
"requires": {
"passport-oauth2": "1.3.x"
},
"dependencies": {
"passport-oauth2": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/passport-oauth2/-/passport-oauth2-1.3.0.tgz",
"integrity": "sha512-AisRXcC7Vx8WSx5C9O8UNib/Y91/IOphs5tImlOMs7T+QTEJdmS+OxgXSskbnDdGQT/8mU5T3yMT6136rBjYXg==",
"requires": {
"oauth": "0.9.x",
"passport-strategy": "1.x.x",
"uid2": "0.0.x"
}
}
"passport-oauth2": "^1.6.1"
}
},
"passport-google-oauth20": {
+2 -2
View File
@@ -28,7 +28,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.21.0",
"@dadosfera/protospack-v2": "3.22.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -52,7 +52,7 @@
"mixpanel": "^0.17.0",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.1.5",
"passport-forcedotcom": "^0.2.0",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
+12 -1
View File
@@ -7,6 +7,7 @@ import {
HttpStatus,
Inject,
UseFilters,
Get,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
@@ -21,7 +22,10 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { RequireAllPermissions } from 'src/authentication/authentication.decorator';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
import { AuthClientService } from './auth.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
@@ -185,4 +189,11 @@ export class AuthController {
return this.authClient.verifyTotp({ accessToken, totp });
}
@Authenticated()
@Get('verify-access-token')
@HttpCode(HttpStatus.OK)
verifyAccessToken() {
return { access_token_status: 'valid' };
}
}
@@ -10,6 +10,7 @@ import {
Post,
Put,
Query,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
@@ -32,9 +33,11 @@ import {
IPreviewResponse,
IUpdateDataRequest,
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
@ApiTags('Catalog')
@Controller('catalog')
@UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated()
export class CatalogController {
logger: DadosferaLogger;
+11 -7
View File
@@ -18,6 +18,7 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { IUpdateDataRequest } from './dtos';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -60,15 +61,11 @@ class CatalogService implements OnModuleInit {
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
return lastValueFrom(
this.catalogWriteService.CreateDataAsset(data, metadata),
).catch((err) => {
throw new HttpException(
err.details,
err.code === 6 ? HttpStatus.CONFLICT : 404,
);
});
);
}
async managePermissions(data: Messages.ManagePermissionRequest, metadata) {
@@ -200,7 +197,14 @@ class CatalogService implements OnModuleInit {
return { data_asset: asset[0] };
}
async updateOneDataAsset({ data_asset_id, customer_id, body, metadata }) {
async updateOneDataAsset(data: {
data_asset_id: string;
customer_id: string;
body: IUpdateDataRequest;
metadata: Metadata;
}) {
const { body, customer_id, data_asset_id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{ id: data_asset_id, changes: JSON.stringify(body) },
+14 -5
View File
@@ -1,6 +1,10 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export class EmbedObject {
@ApiProperty()
url: string;
}
export class IRole {
@ApiProperty()
name: string;
@@ -49,8 +53,8 @@ export class IDataAsset {
data_encrypted?: string;
@ApiPropertyOptional()
num_rows?: number;
@ApiPropertyOptional()
display_name?: string;
@ApiProperty()
display_name: string;
@ApiProperty()
pipeline_id: string;
@ApiProperty()
@@ -75,12 +79,14 @@ export class IDataAsset {
parameters?: string[];
@ApiPropertyOptional()
embedding_params?: object;
@ApiPropertyOptional()
created_at?: string;
@ApiProperty()
created_at: string;
@ApiPropertyOptional()
updated_at?: string;
@ApiProperty({ type: [IComment] })
comments: IComment[];
@ApiPropertyOptional()
embed?: EmbedObject;
}
export class IOneDataAsset {
@@ -127,8 +133,9 @@ export class IUpdateDataRequest {
description: string;
@ApiProperty()
tags: string[];
@ApiPropertyOptional()
embed: EmbedObject;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
display_name: string;
@@ -140,6 +147,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
external_url: string;
@ApiProperty()
location: string;
@ApiPropertyOptional()
embed: EmbedObject;
}
export class IPreview {
+8
View File
@@ -292,6 +292,14 @@ export function EnrichErrorCode(code: string) {
message: 'Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.CATALOG.DATA_ASSET_NOT_FOUND:
return {
statusCode: HttpStatus.NOT_FOUND,
error: 'Não encontramos o ativo solicitado',
message:
'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.INTERNAL:
case ErrorCodes.UNKNOWN:
default:
+4
View File
@@ -67,6 +67,9 @@ const CONNECTION = {
const TRANSFORMATION = {
NOT_FOUND: 'TRANSFORMATION.NOT_FOUND',
};
const CATALOG = {
DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
};
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
@@ -82,6 +85,7 @@ const ErrorCodes = {
INPUT,
TRANSFORMATION,
CONNECTION,
CATALOG,
};
export default ErrorCodes;