Authentication middleware implementation on Gateway + Documentation implementation w/ auth

This commit is contained in:
rodrigo.zamboni
2022-01-27 18:18:31 -03:00
parent 332e0f0d80
commit a824a8768b
34 changed files with 4945 additions and 4060 deletions
+4
View File
@@ -32,6 +32,8 @@
"axios": "^0.25.0",
"dotenv": "^14.2.0",
"grpc": "^1.24.11",
"jsonwebtoken": "^8.5.1",
"jwk-to-pem": "^2.0.5",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.2.0",
@@ -43,6 +45,8 @@
"@nestjs/testing": "^8.0.0",
"@types/express": "^4.17.13",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.8",
"@types/jwk-to-pem": "^2.0.1",
"@types/node": "^16.0.0",
"@types/supertest": "^2.0.11",
"@typescript-eslint/eslint-plugin": "^5.0.0",
+18 -3
View File
@@ -1,4 +1,4 @@
import { Module } from '@nestjs/common';
import { MiddlewareConsumer, Module, NestModule, RequestMethod } from '@nestjs/common';
import { InputGrpcClientModule } from './inputs/input-grpc-client.module';
import { UsersGrpcClientModule } from './users-grpc-client/users-grpc-client.module';
import { OutputsModule } from './outputs/outputs.module';
@@ -12,7 +12,13 @@ import { RestTransformationsModule } from './rest/rest-transformations/rest-tran
import { RestPipelinesModule } from './rest/rest-pipelines/rest-pipelines.module';
import { RestUserModule } from './rest/rest-user/rest-user.module';
import { RestFactoryModule } from './rest/rest-factory/rest-factory.module';
import { LoggerMiddleware } from './middlewares/authentication';
import { RestInputsController } from './rest/rest-inputs/rest-inputs.controller';
import { RestOutputsController } from './rest/rest-outputs/rest-outputs.controller';
import { RestTransformationsController } from './rest/rest-transformations/rest-transformations.controller';
import { RestPipelinesController } from './rest/rest-pipelines/rest-pipelines.controller';
import { RestEnrichmentsController } from './rest/rest-enrichments/rest-enrichments.controller';
import { RestFactoryController } from './rest/rest-factory/rest-factory.controller';
@Module({
imports: [
RestInputsModule, RestOutputsModule, RestEnrichmentsModule, RestTransformationsModule, RestPipelinesModule, RestUserModule, RestFactoryModule,
@@ -25,4 +31,13 @@ import { RestFactoryModule } from './rest/rest-factory/rest-factory.module';
controllers: [],
providers: [],
})
export class AppModule {}
export class AppModule implements NestModule{
configure(consumer: MiddlewareConsumer) {
consumer
.apply(LoggerMiddleware)
.forRoutes(RestInputsController,RestOutputsController,RestTransformationsController,RestPipelinesController,RestEnrichmentsController,RestFactoryController)
}
}
@@ -1,22 +1,13 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateInputGrpcClientDto {
@ApiProperty()
cron:string;
@ApiProperty({
description:"Name of the input",
example:"Test Name"
})
name:string;
@ApiProperty()
plugin:string;
@ApiProperty()
values:string;
@ApiProperty()
operation:string;
}
@@ -15,7 +15,6 @@ import { GrpcClientConfiguration } from './config/grpc-client';
import { TestConnectionDTO } from './dto/test-connection-grpc-client.dto';
import { ApiTags } from '@nestjs/swagger';
@ApiTags('inputs-grpc')
@Controller('grpc/input')
export class InputGrpcClientController implements OnModuleInit {
@Client(GrpcClientConfiguration)
+1 -1
View File
@@ -11,7 +11,7 @@ async function bootstrap() {
.setTitle('Maestro Grpc Documentation')
.setDescription('Documentation for Maestro gateway')
.setVersion('1.0')
.addTag('inputs')
.addBearerAuth()
.build();
const document = SwaggerModule.createDocument(app, config);
+92
View File
@@ -0,0 +1,92 @@
import jwt from 'jsonwebtoken';
// import {decode,verify} from 'jsonwebtoken'
import jwkToPem from 'jwk-to-pem';
import { Request, Response, NextFunction } from 'express';
import axios from 'axios';
import { NestMiddleware } from '@nestjs/common';
// let pems: { [key: string]: {} }[];
export class LoggerMiddleware implements NestMiddleware{
setUp = async (region: string, id: string):Promise<{ [key: string]: {} }[]> => {
const URL = `https://cognito-idp.${region}.amazonaws.com/${id}/.well-known/jwks.json`;
try {
const response = await axios.get(URL);
if (response.status !== 200) {
throw new Error('Deu Ruim');
}
const data = await response.data;
const { keys } = data;
return keys.map((key: any) => {
const modulus = key.n;
const exponent = key.e;
const keyType = key.kty;
const jwk = { kty: keyType, n: modulus, e: exponent };
const pem = jwkToPem(jwk);
const keyId = key.kid;
return { [keyId]: pem };
});
// console.log('got PEMS');
} catch (error) {
// console.log(error);
// console.log('Error! Unable to download JWKs');
}
};
use = async (
request: Request,
response: Response,
next: NextFunction
) => {
const awsRegion = process.env.AWS_REGION
? process.env.AWS_REGION
: 'undefined';
const awsPoolId = process.env.AWS_IDENTITY_POOL_ID
? process.env.AWS_IDENTITY_POOL_ID
: 'undefined';
try {
const pems = await this.setUp(awsRegion, awsPoolId);
const userToken = request.headers.authorization?.split(' ')[1];
if (userToken) {
const user: any = jwt.decode(userToken, { complete: true });
if (user === null) {
response.status(401).end();
return;
}
const { kid } = user.header;
const pem = pems.filter((item: any) => item[kid]);
const pemValue: any = pem[0][kid];
if (!pem) {
response.status(401).end();
return;
}
jwt.verify(userToken, pemValue, (err: any) => {
if (err) {
response.status(401).json(err);
return;
}
next();
});
}
} catch (error) {
response.status(401).send({ message: 'Authentication Failed!' });
}
};
}
-1
View File
@@ -6,7 +6,6 @@ import { UpdateOutputDto } from './dto/update-output.dto';
import { ApiTags } from '@nestjs/swagger';
import { GrpcClientConfiguration } from './config/grpc-client';
@ApiTags('outputs')
@Controller('output')
export class OutputsController implements OnModuleInit{
-2
View File
@@ -4,9 +4,7 @@ import { PipelineGrpcClientService } from './pipelines.service';
import { CreatePipelineDto } from './dto/create-pipeline.dto';
import { UpdatePipelineDto } from './dto/update-pipeline.dto';
import { GrpcClientConfiguration } from './config/grpc-client';
import { ApiTags } from '@nestjs/swagger';
@ApiTags('pipelines')
@Controller('pipeline')
export class PipelinesController implements OnModuleInit{
@@ -2,5 +2,5 @@ export class CreateRestEnrichmentDto {
type:string;
columns:string[]
columns
}
@@ -4,7 +4,9 @@ import { CreateRestEnrichmentDto } from './dto/create-rest-enrichment.dto';
import { UpdateRestEnrichmentDto } from './dto/update-rest-enrichment.dto';
import { AuthenticateController } from 'src/utils/Authenticate';
import axios from 'axios';
import { ApiTags } from '@nestjs/swagger';
@ApiTags('Enrichments')
@Controller('enrichment')
export class RestEnrichmentsController {
constructor(private readonly restEnrichmentsService: RestEnrichmentsService) {}
@@ -4,7 +4,10 @@ import { CreateRestFactoryDto } from './dto/create-rest-factory.dto';
import { UpdateRestFactoryDto } from './dto/update-rest-factory.dto';
import { AuthenticateController } from 'src/utils/Authenticate';
import axios from 'axios';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
@ApiTags('Dag Factory')
@ApiBearerAuth()
@Controller('factory')
export class RestFactoryController {
constructor(private readonly restFactoryService: RestFactoryService) {}
@@ -0,0 +1,17 @@
import { ApiProperty } from "@nestjs/swagger";
export class HTTPInputValues{
@ApiProperty({
description:"Choosen url",
example:"http://testroute/dadosfera/inputs/values"
})
urls:string;
@ApiProperty({
description:"Schedule name",
example:"Test_Schedule"
})
schedule:string
}
@@ -0,0 +1,53 @@
import { ApiProperty } from "@nestjs/swagger";
export class JDBCInputvalues{
@ApiProperty({
description:"JDBC user",
example:"airflow"
})
jdbc_user: string;
@ApiProperty({
description:"JDBC password",
example:"password123"
})
jdbc_password: string;
@ApiProperty({
description:"Database",
example:"airflow"
})
database: string;
@ApiProperty({
description:"Connection endpoint",
example:"dadosfera-postgres-db.test123.us-east"
})
endpoint: string;
@ApiProperty({
description:"Schema name",
example:"cadastros"
})
schema?:string;
@ApiProperty({
description:"Tables names",
example:["pessoas", "clients"]
})
table: string[];
@ApiProperty({
description:"Port",
example:"5432"
})
port: string;
@ApiProperty({
description:"Engine name",
example:"postgres"
})
engine: string;
}
@@ -1,22 +1,64 @@
import { ApiProperty } from "@nestjs/swagger";
import { CronDTO } from "./cron.dto";
import { HTTPInputValues } from "./HTTPInputValues.dto";
import { JDBCInputvalues } from "./JDBCInputValues.dto";
import { S3InputValues } from "./s3InputValues.dto";
import { SalesforceInputValuesDTO } from "./salesforceInputValues.dto";
export class CreateRestInputDto {
@ApiProperty()
cron:string;
@ApiProperty({
description:"Input CRON",
example:{
hours: ["3", "7"],
hour_interval: true,
hour_resourse: true,
hour_resourse_value: 3,
month_day: ["3"],
month_day_interval: true,
month_day_resourse: false,
month_day_resourse_value: null,
month: ["3"],
month_interval: false,
month_resourse: false,
month_resourse_value: null,
week_days: [],
week_days_interval: false,
week_days_resourse: false,
week_days_resourse_value: null
}
})
cron:CronDTO;
@ApiProperty({
description:"Name of the input",
example:"Test Name"
example:"Test_Input"
})
name:string;
@ApiProperty()
@ApiProperty({
description:"Plugin",
example:"s3"
})
plugin:string;
@ApiProperty()
values:string;
@ApiProperty({
description:"Values for choosen input",
example:{
jdbc_user: "user123",
jdbc_password: "password123",
database: "default_database",
endpoint: "dadosfera-postgres-db.test123.us-east-1.rds.amazonaws.com",
table: "test_table",
port: "5432",
engine: "postgresql"
}
})
values: SalesforceInputValuesDTO|HTTPInputValues|S3InputValues|JDBCInputvalues;
@ApiProperty()
@ApiProperty({
description:"Operation name",
example:"input"
})
operation:string;
}
+102
View File
@@ -0,0 +1,102 @@
import { ApiProperty } from "@nestjs/swagger";
export class CronDTO{
@ApiProperty({
description:"Hours to the cron",
example:["3","7"]
})
hours: string[];
@ApiProperty({
description:"Boolean for hour interval",
example:true
})
hour_interval: boolean;
@ApiProperty({
description:"Boolean for hour resourse",
example:true
})
hour_resourse: boolean;
@ApiProperty({
description:"Value for hour resourse",
example:3
})
hour_resourse_value: number;
@ApiProperty({
description:"Month day to the cron",
example:["3"]
})
month_day: string[];
@ApiProperty({
description:"Boolean for month day interval",
example:true
})
month_day_interval: boolean;
@ApiProperty({
description:"Boolean for month day resourse",
example:false
})
month_day_resourse: boolean;
@ApiProperty({
description:"Value for month day resourse",
example:null
})
month_day_resourse_value:number|null;
@ApiProperty({
description:"Month to the cron",
example:["3"]
})
month: string[];
@ApiProperty({
description:"Boolean from month interval",
example:false
})
month_interval: boolean;
@ApiProperty({
description:"Boolean from month resourse",
example:false
})
month_resourse: boolean;
@ApiProperty({
description:"Value to month resourse",
example:null
})
month_resourse_value:number|null;
@ApiProperty({
description:"Week days to the cron",
example:[]
})
week_days: string[];
@ApiProperty({
description:"Boolean for the week days",
example:false
})
week_days_interval: boolean;
@ApiProperty({
description:"Boolean for week day resourse",
example:false
})
week_days_resourse: boolean;
@ApiProperty({
description:"Hours to the cron",
example:null
})
week_days_resourse_value:number|null
}
@@ -0,0 +1,50 @@
import { ApiProperty } from "@nestjs/swagger";
export class S3InputValues{
@ApiProperty({
description:"file to extract path",
example:"s3_connector/testfile.csv"
})
file_to_extract: string;
@ApiProperty({
description:"target bucket path",
example:"dadosfera-test"
})
target_bucket?: string;
@ApiProperty({
description:"client bucket path",
example:"dadosfera-test"
})
client_bucket: string;
@ApiProperty({
description:"Format to the file",
example:"csv"
})
format_file: string;
@ApiProperty({
description:"Clients AWS Access Key Id",
example:"AKIBN5B19D3NGZI"
})
client_aws_access_key_id: string;
@ApiProperty({
description:"Clients AWS Secret Access Key Id",
example:"ct5R1EjrVC78bvaJ636gG90O5NqOrOUoq4Ri"
})
client_aws_secret_access_key : string;
@ApiProperty({
description:"file params format",
example:{sep: "," , header: "true"}
})
format_file_params?: {
sep: string,
header: string
}
}
@@ -0,0 +1,27 @@
import { ApiProperty } from "@nestjs/swagger";
export class SalesforceInputValuesDTO{
@ApiProperty({
description:"Salesforce username",
example:"dadosfera@devs.com"
})
username_salesforce: string;
@ApiProperty({
description:"Salesforce password",
example:"Salesforce@test123"
})password_salesforce: string;
@ApiProperty({
description:"Salesforce Security Token",
example:"A67Tdg7qHUvhZDOFjWwtPMjmQ"
})security_token_salesforce: string;
@ApiProperty({
description:"Salesforce Query",
example:"SELECT * FROM Users"
})query_salesforce: string;
}
@@ -0,0 +1,34 @@
import { ApiProperty } from "@nestjs/swagger";
import { HTTPInputValues } from "./HTTPInputValues.dto";
import { JDBCInputvalues } from "./JDBCInputValues.dto";
import { S3InputValues } from "./s3InputValues.dto";
import { SalesforceInputValuesDTO } from "./salesforceInputValues.dto";
export class testConnectionDTO{
@ApiProperty({
description:"Input Plugin Name",
example:"mysql"
})
plugin:string;
@ApiProperty({
description:"Plugin name",
example:{
plugin: "mysql",
values: {
jdbc_user: "test",
jdbc_password: "test1234",
database: "users",
endpoint: "dadosfera.test123.us-east-1.rds.amazonaws.com",
schema: "user",
port: "5432",
engine: "mysql"
}
}
})
values:HTTPInputValues|SalesforceInputValuesDTO|S3InputValues|JDBCInputvalues
}
+15 -6
View File
@@ -1,19 +1,28 @@
import axios from 'axios';
require("dotenv").config();
import { RestInputsService } from './rest-inputs.service';
import { CreateInputGrpcClientDto } from 'src/inputs/dto/create-input-grpc-client.dto';
import { UpdateInputGrpcClientDto } from 'src/inputs/dto/update-input-grpc-client.dto';
import { Controller, Get, Post, Body, Patch, Param, Delete, Put, Headers } from '@nestjs/common';
import { AuthenticateController } from 'src/utils/Authenticate';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
import { CreateRestInputDto } from './dto/create-rest-input.dto';
import { UpdateRestInputDto } from './dto/update-rest-input.dto';
import { testConnectionDTO } from './dto/testConnection.dto';
@ApiTags('Inputs')
@ApiBearerAuth()
@Controller('input')
export class RestInputsController {
constructor(private readonly restInputsService: RestInputsService) {}
@Get('/test-auth/')
async testauth(){
return "Auth Success"
}
@Post('/test-connection/')
async testConnection(@Body() body){
async testConnection(@Body() body:testConnectionDTO){
const authController = new AuthenticateController()
const accessToken = await authController.authenticate()
@@ -33,7 +42,7 @@ export class RestInputsController {
}
@Post()
async create(@Body() body: CreateInputGrpcClientDto){
async create(@Body() body: CreateRestInputDto){
const authController = new AuthenticateController()
const accessToken = await authController.authenticate()
@@ -92,7 +101,7 @@ export class RestInputsController {
@Put(':id')
async update(
@Body() body: UpdateInputGrpcClientDto,
@Body() body: UpdateRestInputDto,
@Param() id: string,
){
const authController = new AuthenticateController()
@@ -1,7 +1,32 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateRestOutputDto {
@ApiProperty({
description:"Output name",
example:"Test Output"
})
name:string;
@ApiProperty({
description:"Plugin name",
example:"Test Plugin"
})
plugin:string;
values:string;
@ApiProperty({
description:"Plugin values",
example:{
prefix:"test"
}
})
values:{
prefix:string
};
@ApiProperty({
description:"Operation name",
example:"output"
})
operation:string;
}
@@ -4,7 +4,10 @@ import { CreateRestOutputDto } from './dto/create-rest-output.dto';
import { UpdateRestOutputDto } from './dto/update-rest-output.dto';
import { AuthenticateController } from 'src/utils/Authenticate';
import axios from 'axios';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
@ApiTags('Outputs')
@ApiBearerAuth()
@Controller('output')
export class RestOutputsController {
constructor(private readonly restOutputsService: RestOutputsService) {}
@@ -1 +1,59 @@
export class CreateRestPipelineDto {}
import { ApiProperty } from "@nestjs/swagger";
export class CreateRestPipelineDto {
@ApiProperty({
description:"Pipeline Input",
example:{
id:"ac8f828a-708d-4f28-a5b4-0170eaf96843"
}
})
input:{
id:string
}
@ApiProperty({
description:"Pipeline Transformations",
example:[
{
id:"b2579a1a-2874-47f6-b609-7568a2930468"
},
{
id:"6bac1a1a-3037-474f-8211-c3ceb8ea04e8"
}
]
})
transformations:[
{
id:string
}
]
@ApiProperty({
description:"Pipeline Output",
example:{
id:"06f9ce35-d9c0-4f95-a49b-c09ea00b8e48"
}
})
output:{
id:string
}
@ApiProperty({
description:"Pipeline Tags",
example:[]
})
tags:string[]
@ApiProperty({
description:"Pipeline Name",
example:"Dadosfera Pipeline"
})
name:string
@ApiProperty({
description:"Pipeline Description",
example:"Example pipeline"
})
description:string
}
@@ -4,7 +4,10 @@ import { CreateRestPipelineDto } from './dto/create-rest-pipeline.dto';
import { UpdateRestPipelineDto } from './dto/update-rest-pipeline.dto';
import { AuthenticateController } from 'src/utils/Authenticate';
import axios from 'axios';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
@ApiTags('Pipelines')
@ApiBearerAuth()
@Controller('pipeline')
export class RestPipelinesController {
constructor(private readonly restPipelinesService: RestPipelinesService) {}
@@ -1 +1,52 @@
export class CreateRestTransformationDto {}
import { ApiProperty } from "@nestjs/swagger";
export class Transformation{
transformations:[{
type:string;
params:[{
base_column:string,
column_name:string
}]
}]
}
export class UpdateTransformation{
transformations:[{
inputSource:string;
table:string;
type:string;
params:[{
base_column:string,
column_name:string
}]
}]
}
export class CreateRestTransformationDto {
@ApiProperty({
description:"Array of transformations",
example:[
{
type:"hide_full_name",
params:[
{
base_column:"full_name",
column_name:"hiden_full_name"
}
]
},
{
type: "hide_cpf",
params: [
{
base_column: "cpf",
column_name: "cpf_hashed"
}
]
},
]
})
transformations: Transformation | UpdateTransformation;
}
@@ -4,7 +4,10 @@ import { CreateRestTransformationDto } from './dto/create-rest-transformation.dt
import { UpdateRestTransformationDto } from './dto/update-rest-transformation.dto';
import { AuthenticateController } from 'src/utils/Authenticate';
import axios from 'axios';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
@ApiTags('Transformations')
@ApiBearerAuth()
@Controller('transformation')
export class RestTransformationsController {
constructor(private readonly restTransformationsService: RestTransformationsService) {}
+40 -1
View File
@@ -1 +1,40 @@
export class CreateRestUserDto {}
import { ApiProperty } from "@nestjs/swagger";
export class CreateRestUserDto {
@ApiProperty({
description:"User Email",
example:"dev@dadosfera.ai"
})
email: string;
@ApiProperty({
description:"User Password",
example:"dadospassword123"
})
password: string;
@ApiProperty({
description:"User Username",
example:"dev_fera"
})
username: string;
@ApiProperty({
description:"User name",
example:"Fulano da Silva"
})
name: string;
@ApiProperty({
description:"Customer id",
example:"dadosfera"
})
customer_id: string;
@ApiProperty({
description:"User rules",
example:"['admin']"
})
rules: string;
}
+10
View File
@@ -0,0 +1,10 @@
import { ApiProperty } from "@nestjs/swagger";
export class ResendCodeDTO{
@ApiProperty({
description:"User username",
example:"dev_fera"
})
username:string
}
+18
View File
@@ -0,0 +1,18 @@
import { ApiProperty } from "@nestjs/swagger";
export class UserAuthDTO{
@ApiProperty({
description:"User username",
example:"dev_fera"
})
username:string;
@ApiProperty({
description:"User password",
example:"dadospassword123"
})
password:string;
}
@@ -0,0 +1,17 @@
import { ApiProperty } from "@nestjs/swagger";
export class UserConfirmationDTO{
@ApiProperty({
description:"User username",
example:"dev_fera"
})
username:string;
@ApiProperty({
description:"Received Confirmation Code",
example:"315246"
})
confirmationCode:string;
}
+9 -3
View File
@@ -4,7 +4,13 @@ import { CreateRestUserDto } from './dto/create-rest-user.dto';
import { UpdateRestUserDto } from './dto/update-rest-user.dto';
import { AuthenticateController } from 'src/utils/Authenticate';
import axios from 'axios';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
import { UserAuthDTO } from './dto/userAuth.dto';
import { UserConfirmationDTO } from './dto/userConfirmation.dto';
import { ResendCodeDTO } from './dto/resendCode.dto';
@ApiTags('Users')
@ApiBearerAuth()
@Controller('user')
export class RestUserController {
constructor(private readonly restUserService: RestUserService) {}
@@ -27,7 +33,7 @@ export class RestUserController {
}
@Post('/auth')
async auth(@Body() body: {username:string, password:string}) {
async auth(@Body() body: UserAuthDTO) {
const connectionResponse = await axios.post(
@@ -44,7 +50,7 @@ export class RestUserController {
}
@Post('/confirm')
async confirm(@Body() body: {username:string, confirmationCode:string}) {
async confirm(@Body() body: UserConfirmationDTO) {
const connectionResponse = await axios.post(
@@ -61,7 +67,7 @@ export class RestUserController {
}
@Post('/resendCode')
async resendCode(@Body() body: {username:string}) {
async resendCode(@Body() body: ResendCodeDTO) {
const connectionResponse = await axios.post(
@@ -6,7 +6,6 @@ import { UpdateTransformationDto } from './dto/update-transformation.dto';
import { ApiTags } from '@nestjs/swagger';
import { GrpcClientConfiguration } from './config/grpc-client';
@ApiTags('transformations')
@Controller('transformation')
export class TransformationsController implements OnModuleInit {
+1 -1
View File
File diff suppressed because one or more lines are too long
+1
View File
@@ -3,6 +3,7 @@
"module": "commonjs",
"declaration": true,
"removeComments": true,
"esModuleInterop": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"allowSyntheticDefaultImports": true,
+4235 -4020
View File
File diff suppressed because it is too large Load Diff