Compare commits

...
18 Commits
Author SHA1 Message Date
Rodrigo Zamboni 849b979e63 Merge pull request #81 from dadosfera/feat/CatalogFixes
Feat/catalog fixes
2022-04-28 14:19:31 -03:00
rodrigo.zamboni 28580782b1 FIX: catalog routes fix 2022-04-28 14:14:03 -03:00
rodrigo.zamboni c997e91912 FIX: fix table-metadata find one route 2022-04-28 12:59:35 -03:00
rodrigo.zamboni 22f493d48a FIX: fixing missing id on delete routes 2022-04-28 12:12:37 -03:00
rodrigo.zamboni 035213824b FIX: fixing missing id on delete routes 2022-04-28 12:00:19 -03:00
rodrigo.zamboni 1ffa9ade51 FIX: CI 2022-04-28 10:53:17 -03:00
Rodrigo Zamboni c91aa57344 Merge pull request #78 from dadosfera/beta
Beta
2022-04-27 17:36:54 -03:00
Rodrigo Zamboni 077a95cdbd Merge pull request #77 from dadosfera/alpha
Alpha
2022-04-27 17:16:09 -03:00
Rodrigo Zamboni f39108772b Merge pull request #76 from dadosfera/NewInput
FIX: fix pipeline activate route method
2022-04-27 16:57:56 -03:00
rodrigo.zamboni cdd8beee8f FIX: fix pipeline activate route method 2022-04-27 16:56:03 -03:00
rodrigo.zamboni 14331ed229 FIX: fix pipeline activate route method 2022-04-27 16:54:48 -03:00
Victor Radael 5f3db0cb43 Merge pull request #75 from dadosfera/beta
Beta -> Main
2022-04-27 11:42:18 -03:00
Rodrigo Zamboni 28f304c47c Merge pull request #74 from dadosfera/alpha
Alpha
2022-04-26 17:41:57 -03:00
Rodrigo Zamboni 0c1b62b2aa Merge pull request #73 from dadosfera/NewInput
FIX: Fix get-columns route method
2022-04-26 17:25:11 -03:00
rodrigo.zamboni 13f7e9d812 FIX: Fix get-columns route method 2022-04-26 17:19:18 -03:00
Gabriel Amorim 841dcfef06 Merge pull request #72 from dadosfera/alpha
Alpha
2022-04-26 16:30:31 -03:00
Gabriel Amorim f41ddc1483 Merge pull request #71 from dadosfera/NewInput
UPDATE: Implemented new Input format
2022-04-26 16:19:19 -03:00
rodrigo.zamboni 0a2c7c0a38 UPDATE: Implemented new Input format 2022-04-26 16:15:54 -03:00
10 changed files with 201 additions and 907 deletions
+7
View File
@@ -0,0 +1,7 @@
Resources:
AWSEBAutoScalingLaunchConfiguration:
Properties:
MetadataOptions:
HttpEndpoint: enabled
HttpPutResponseHopLimit: 2
HttpTokens: required
+23 -852
View File
File diff suppressed because it is too large Load Diff
-1
View File
@@ -33,7 +33,6 @@
"@victorradael/protospack": "^1.5.5", "@victorradael/protospack": "^1.5.5",
"axios": "^0.25.0", "axios": "^0.25.0",
"dotenv": "^14.2.0", "dotenv": "^14.2.0",
"grpc": "^1.24.11",
"helmet": "^5.0.2", "helmet": "^5.0.2",
"jsonwebtoken": "^8.5.1", "jsonwebtoken": "^8.5.1",
"jwk-to-pem": "^2.0.5", "jwk-to-pem": "^2.0.5",
+48 -22
View File
@@ -1,16 +1,16 @@
import { OnModuleInit, Inject } from '@nestjs/common'; import { OnModuleInit, Inject } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices'; import { ClientGrpc } from '@nestjs/microservices';
import { InputService } from '@victorradael/protospack'; import {
InputNewCreateRequest,
InputService,
TestConnectionGetColumnsRequest,
TestConnectionRequest,
} from '@victorradael/protospack';
import { import {
objectCamelToSnake, objectCamelToSnake,
objectSnakeToCamel, objectSnakeToCamel,
} from 'src/utils/CaseConverter'; } from 'src/utils/CaseConverter';
import { import { IIdRequest, UpdateInputRequest } from './interfaces';
ICreateInputRequest,
IIdRequest,
ITestConnectionRequest,
UpdateInputRequest,
} from './interfaces';
export class InputsClientService implements OnModuleInit { export class InputsClientService implements OnModuleInit {
private inputService: InputService; private inputService: InputService;
@@ -23,22 +23,24 @@ export class InputsClientService implements OnModuleInit {
this.grpcClient.getService<InputService>('InputService'); this.grpcClient.getService<InputService>('InputService');
} }
async create(createInputDto: ICreateInputRequest) { async create(createInputDto: InputNewCreateRequest) {
console.log('InputClientService', 'Create'); console.log('InputClientService', 'Create');
const createInputResponse = await new Promise((resolve, reject) => { const createInputResponse = await new Promise((resolve, reject) => {
this.inputService.Create(objectSnakeToCamel(createInputDto)).subscribe({ this.inputService
next(x) { .NewCreate(objectSnakeToCamel(createInputDto))
resolve(objectCamelToSnake(x)); .subscribe({
}, next(x) {
error(err) { resolve(objectCamelToSnake(x));
console.log('Observable Error'); },
reject(err); error(err) {
}, console.log('Observable Error');
complete() { reject(err);
console.log('done'); },
}, complete() {
}); console.log('done');
},
});
}) })
.then((res) => res) .then((res) => res)
.catch((err) => { .catch((err) => {
@@ -147,10 +149,10 @@ export class InputsClientService implements OnModuleInit {
return removeInputResponse; return removeInputResponse;
} }
async testConnection(data: ITestConnectionRequest) { async testConnection(data: TestConnectionRequest) {
console.log('InputClientService', 'TestConnection'); console.log('InputClientService', 'TestConnection');
const testConnectionResponse = await new Promise((resolve, reject) => { const testConnectionResponse = await new Promise((resolve, reject) => {
this.inputService.TestConnection(objectSnakeToCamel(data)).subscribe({ this.inputService.NewTestConnection(objectSnakeToCamel(data)).subscribe({
next(x) { next(x) {
resolve(objectCamelToSnake(x)); resolve(objectCamelToSnake(x));
}, },
@@ -170,4 +172,28 @@ export class InputsClientService implements OnModuleInit {
return testConnectionResponse; return testConnectionResponse;
} }
async getColumns(data: TestConnectionGetColumnsRequest) {
console.log('InputClientService', 'TestConnection/Get-Columns');
const getColumnsResponse = await new Promise((resolve, reject) => {
this.inputService.GetColumns(objectSnakeToCamel(data)).subscribe({
next(x) {
resolve(objectCamelToSnake(x));
},
error(err) {
console.log('Observable Error');
reject(err);
},
complete() {
console.log('done');
},
});
})
.then((res) => res)
.catch((err) => {
throw new Error(err);
});
return getColumnsResponse;
}
} }
+50 -17
View File
@@ -1,4 +1,12 @@
import { Body, Controller, Delete, Get, Param, Post } from '@nestjs/common'; import {
Body,
Controller,
Delete,
Get,
Param,
Post,
Query,
} from '@nestjs/common';
import { CatalogService } from './catalog.service'; import { CatalogService } from './catalog.service';
@Controller('catalog') @Controller('catalog')
@@ -33,7 +41,7 @@ export class CatalogController {
const { id } = params; const { id } = params;
const catalogService = new CatalogService(); const catalogService = new CatalogService();
const res = await catalogService.dataAppsOne(body, id); const res = await catalogService.dataAppsOne(id, body);
return res; return res;
} }
@@ -56,30 +64,33 @@ export class CatalogController {
const catalogService = new CatalogService(); const catalogService = new CatalogService();
const res = await catalogService.getOneDashboardMetabase(body, id); const res = await catalogService.getOneDashboardMetabase(id, body);
return res; return res;
} }
@Get('table-metadata') @Get('table-metadata')
async getAllTableMetadata(@Body() body) { async getAllTableMetadata(@Body() body, @Query() query) {
console.log(`/catalog`, 'ON GET ALL TABLES METADATA ROUTE'); console.log(`/catalog`, 'ON GET ALL TABLES METADATA ROUTE');
const catalogService = new CatalogService(); const catalogService = new CatalogService();
if (!query) {
const res = await catalogService.getAllTableMetadata(body); const res = await catalogService.getAllTableMetadata(body);
return res;
return res; } else {
const res = await catalogService.getOneTableMetadata(body, query);
return res;
}
} }
@Delete('table-metadata') @Delete('table-metadata/:id')
async deleteOneTableMetadata(@Param() params, @Body() body) { async deleteOneTableMetadata(@Param() params, @Body() body) {
const { id } = params; const { id } = params;
console.log(`/catalog`, 'ON DELETE ONE TABLE METADATA ROUTE'); console.log(`/catalog`, 'ON DELETE ONE TABLE METADATA ROUTE');
const catalogService = new CatalogService(); const catalogService = new CatalogService();
const res = await catalogService.deleteOneTableMetadata(body, id); const res = await catalogService.deleteOneTableMetadata(id, body);
return res; return res;
} }
@@ -95,14 +106,14 @@ export class CatalogController {
return res; return res;
} }
@Delete('column-metadata') @Delete('column-metadata/:id')
async deleteOneColumnMetadata(@Param() params, @Body() body) { async deleteOneColumnMetadata(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON DELETE ONE COLUMN METADATA ROUTE'); console.log(`/catalog`, 'ON DELETE ONE COLUMN METADATA ROUTE');
const { id } = params; const { id } = params;
const catalogService = new CatalogService(); const catalogService = new CatalogService();
const res = await catalogService.deleteOneColumnMetadata(body, id); const res = await catalogService.deleteOneColumnMetadata(id, body);
return res; return res;
} }
@@ -162,6 +173,17 @@ export class CatalogController {
return res; return res;
} }
@Post('data-rating')
async createDataRating(@Body() body) {
console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
const catalogService = new CatalogService();
const res = await catalogService.createDataRating(body);
return res;
}
@Get('summary-rating/:id') @Get('summary-rating/:id')
async getSummaryRating(@Param() params, @Body() body) { async getSummaryRating(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON GET SUMMARY RATING ROUTE'); console.log(`/catalog`, 'ON GET SUMMARY RATING ROUTE');
@@ -185,6 +207,17 @@ export class CatalogController {
return res; return res;
} }
@Post('data-comment')
async createDataComment(@Body() body) {
console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
const catalogService = new CatalogService();
const res = await catalogService.createDataComment(body);
return res;
}
@Get('data-review') @Get('data-review')
async getDataReview(@Body() body) { async getDataReview(@Body() body) {
console.log(`/catalog`, 'ON GET DATA REVIEW ROUTE'); console.log(`/catalog`, 'ON GET DATA REVIEW ROUTE');
@@ -225,7 +258,7 @@ export class CatalogController {
const catalogService = new CatalogService(); const catalogService = new CatalogService();
const res = await catalogService.deleteTags(body, id); const res = await catalogService.deleteTags(id, body);
return res; return res;
} }
@@ -252,14 +285,14 @@ export class CatalogController {
return res; return res;
} }
@Delete('table-tags') @Delete('table-tags/:id')
async deleteTableTags(@Param() params, @Body() body) { async deleteTableTags(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE'); console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE');
const { id } = params; const { id } = params;
const catalogService = new CatalogService(); const catalogService = new CatalogService();
const res = await catalogService.deleteTableTags(body, id); const res = await catalogService.deleteTableTags(id, body);
return res; return res;
} }
@@ -275,14 +308,14 @@ export class CatalogController {
return res; return res;
} }
@Delete('table-rules') @Delete('table-rules/:id')
async deleteTableRules(@Param() params, @Body() body) { async deleteTableRules(@Param() params, @Body() body) {
console.log(`/catalog`, 'ON DELETE TABLE RULES ROUTE'); console.log(`/catalog`, 'ON DELETE TABLE RULES ROUTE');
const { id } = params; const { id } = params;
const catalogService = new CatalogService(); const catalogService = new CatalogService();
const res = await catalogService.deleteTableRules(body, id); const res = await catalogService.deleteTableRules(id, body);
return res; return res;
} }
+40 -1
View File
@@ -51,7 +51,7 @@ class CatalogService {
return data; return data;
} }
async getOneDashboardMetabase(body, id) { async getOneDashboardMetabase(id, body) {
const customer = const customer =
body.info.customer.toLowerCase() === 'dadosfera' body.info.customer.toLowerCase() === 'dadosfera'
? `` ? ``
@@ -75,6 +75,19 @@ class CatalogService {
return data; return data;
} }
async getOneTableMetadata(body, params) {
const customer =
body.info.customer.toLowerCase() === 'dadosfera'
? ``
: `-${body.info.customer.toLowerCase()}`;
const { data } = await axios.get(
`${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-metadata/`,
{ params: params },
);
return data;
}
async deleteOneTableMetadata(id, body) { async deleteOneTableMetadata(id, body) {
const customer = const customer =
body.info.customer.toLowerCase() === 'dadosfera' body.info.customer.toLowerCase() === 'dadosfera'
@@ -171,6 +184,19 @@ class CatalogService {
return data; return data;
} }
async createDataRating(body) {
const customer =
body.info.customer.toLowerCase() === 'dadosfera'
? ``
: `-${body.info.customer.toLowerCase()}`;
const { data } = await axios.post(
`${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-rating/`,
body,
);
return data;
}
async getSummaryRating(id, body) { async getSummaryRating(id, body) {
const customer = const customer =
body.info.customer.toLowerCase() === 'dadosfera' body.info.customer.toLowerCase() === 'dadosfera'
@@ -194,6 +220,19 @@ class CatalogService {
return data; return data;
} }
async createDataComment(body) {
const customer =
body.info.customer.toLowerCase() === 'dadosfera'
? ``
: `-${body.info.customer.toLowerCase()}`;
const { data } = await axios.post(
`${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-comment/`,
body,
);
return data;
}
async getDataReview(body) { async getDataReview(body) {
const customer = const customer =
body.info.customer.toLowerCase() === 'dadosfera' body.info.customer.toLowerCase() === 'dadosfera'
+16 -5
View File
@@ -9,10 +9,8 @@ import {
} from '@nestjs/common'; } from '@nestjs/common';
import { InputsService } from './inputs.service'; import { InputsService } from './inputs.service';
import { InputsClientService } from 'src/clients/inputs/client.service'; import { InputsClientService } from 'src/clients/inputs/client.service';
import { import { UpdateInputRequest } from 'src/clients/inputs/interfaces';
ICreateInputRequest, import { InputNewCreateRequest } from '@victorradael/protospack';
UpdateInputRequest,
} from 'src/clients/inputs/interfaces';
@Controller('inputs') @Controller('inputs')
export class InputsController { export class InputsController {
@@ -28,8 +26,21 @@ export class InputsController {
return response; return response;
} }
@Post('/test-connection/get-columns')
async getColumns(@Body() data) {
console.log(
`/test-connection/get-columns`,
'ON TEST CONNECTION GET COLUMNS ROUTE',
);
const inputService = new InputsService(this.inputsClientService);
const response = await inputService.getColumns(data);
return response;
}
@Post() @Post()
async create(@Body() createInputDto: ICreateInputRequest) { async create(@Body() createInputDto: InputNewCreateRequest) {
console.log(`/input`, 'ON CREATE ROUTE'); console.log(`/input`, 'ON CREATE ROUTE');
const inputService = new InputsService(this.inputsClientService); const inputService = new InputsService(this.inputsClientService);
+14 -6
View File
@@ -1,16 +1,13 @@
import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { Body, HttpException, HttpStatus, Injectable } from '@nestjs/common';
import { InputNewCreateRequest } from '@victorradael/protospack';
import { InputsClientService } from 'src/clients/inputs/client.service'; import { InputsClientService } from 'src/clients/inputs/client.service';
import { import { IIdRequest, Info } from 'src/clients/inputs/interfaces';
ICreateInputRequest,
IIdRequest,
Info,
} from 'src/clients/inputs/interfaces';
@Injectable() @Injectable()
export class InputsService { export class InputsService {
constructor(private inputClient: InputsClientService) {} constructor(private inputClient: InputsClientService) {}
async create(@Body() data: ICreateInputRequest) { async create(@Body() data: InputNewCreateRequest) {
try { try {
const createInputResponse = await this.inputClient.create(data); const createInputResponse = await this.inputClient.create(data);
@@ -74,4 +71,15 @@ export class InputsService {
throw new HttpException(err.message, HttpStatus.NOT_FOUND); throw new HttpException(err.message, HttpStatus.NOT_FOUND);
} }
} }
async getColumns(data) {
try {
const testConnectionGetColumnsResponse =
await this.inputClient.getColumns(data);
return testConnectionGetColumnsResponse;
} catch (err) {
throw new HttpException(err.message, HttpStatus.NOT_FOUND);
}
}
} }
@@ -14,13 +14,13 @@ import { PipelinesService } from './pipelines.service';
export class PipelinesController { export class PipelinesController {
constructor(private pipelinesClientService: PipelinesClientService) {} constructor(private pipelinesClientService: PipelinesClientService) {}
@Get('start/:id') @Post('start/:id')
async activate(@Param() params, @Body() body) { async activate(@Param() params, @Body() body) {
const { id } = params; const { id } = params;
const { info } = body; const { info } = body;
console.log( console.log(
process.env.DEV_URL + `/transformation/start/${id}`, process.env.DEV_URL + `/pipeline/start/${id}`,
'ON START PIPELINE ROUTE', 'ON START PIPELINE ROUTE',
); );
+1 -1
View File
File diff suppressed because one or more lines are too long