Compare commits

...
14 Commits
20 changed files with 3002 additions and 9964 deletions
+6 -10
View File
@@ -52,7 +52,7 @@ jobs:
id: semantic
with:
extra_plugins: |
conventional-changelog-eslint@4.0.0
conventional-changelog-eslint@4
branches: |
[
'main'
@@ -87,10 +87,6 @@ jobs:
run: |
python3 -m pip install --upgrade pip
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI
run: |
python3 -m pip install awscli --upgrade
@@ -115,8 +111,8 @@ jobs:
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker-compose -f build.docker-compose.yml build
docker-compose -f build.docker-compose.yml push
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
- name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
@@ -132,15 +128,15 @@ jobs:
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker-compose -f build.docker-compose.dockerhub.yml build
docker-compose -f build.docker-compose.dockerhub.yml push
docker compose -f build.docker-compose.dockerhub.yml build
docker compose -f build.docker-compose.dockerhub.yml push
- name: Create ZIP file to Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
NODE_EXPORTER_URL: ${{format('{0}.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest', steps.aws.outputs.aws-account-id)}} # ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
+8 -2
View File
@@ -6,10 +6,16 @@ on:
jobs:
test:
runs-on: self-hosted
runs-on: [self-hosted, prd]
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@v4
id: aws
with:
aws-region: us-east-1
- name: Build
env:
+1 -1
View File
@@ -1 +1 @@
18.10.0
18.17
+1 -1
View File
@@ -12,7 +12,7 @@
"start:debug"
],
"runtimeExecutable": "npm",
"runtimeVersion": "18.10.0",
"runtimeVersion": "18.17",
"skipFiles": [
"<node_internals>/**"
],
+3 -3
View File
@@ -1,5 +1,5 @@
# install all dependencies
FROM node:18.10-alpine as packages
FROM node:18.17-alpine as packages
WORKDIR /app
COPY package.json package-lock.json ./
RUN apk add --no-cache aws-cli \
@@ -7,12 +7,12 @@ RUN apk add --no-cache aws-cli \
&& npm install
# unit test specific build
FROM node:18.10-alpine as test
FROM node:18.17-alpine as test
WORKDIR /app
COPY --from=packages /app/node_modules ./node_modules
COPY . ./
FROM node:18.10-alpine
FROM node:18.17-alpine
WORKDIR /app
COPY . /app/
COPY --from=packages /app/node_modules ./node_modules
+1 -1
View File
@@ -12,7 +12,7 @@ These instructions will allow you to get a working copy of the project on your l
### 📋 Requirements
- [NodeJS v18.10.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
- [NodeJS v18.17 / NPM v9.6.7](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
- Create your Access Key on the "Security credentials" menu
- Set the Access Key on your local development machine
+191 -1
View File
@@ -133,7 +133,32 @@
},
"get": {
"operationId": "ConnectionController_getAllConnections",
"parameters": [],
"parameters": [
{
"name": "search",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
@@ -1775,6 +1800,118 @@
]
}
},
"/customers/{id}/links": {
"get": {
"operationId": "CustomersController_getCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinksResponse"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
}
]
},
"put": {
"operationId": "CustomersController_setCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinkRequest"
}
}
}
},
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/customers/token": {
"get": {
"operationId": "CustomersController_getCustomerToken",
"parameters": [
{
"name": "exp",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/health": {
"get": {
"operationId": "HealthController_check",
@@ -2072,6 +2209,9 @@
"ConnectionToCatalogDto": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
@@ -3163,6 +3303,56 @@
"updated_at",
"updated_by"
]
},
"CustomerLink": {
"type": "object",
"properties": {
"href": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"iconSrc": {
"type": "string"
}
},
"required": [
"href",
"name",
"description"
]
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
}
}
}
+78 -17
View File
@@ -582,7 +582,32 @@
},
"get": {
"operationId": "ConnectionController_getAllConnections",
"parameters": [],
"parameters": [
{
"name": "search",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
@@ -5001,7 +5026,16 @@
"/customers/{id}/links": {
"get": {
"operationId": "CustomersController_getCustomerLinks",
"parameters": [],
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
@@ -5025,27 +5059,29 @@
},
"put": {
"operationId": "CustomersController_setCustomerLinks",
"parameters": [],
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerDto"
"$ref": "#/components/schemas/CustomerLinkRequest"
}
}
}
},
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
"description": ""
}
},
"tags": [
@@ -5078,7 +5114,7 @@
"200": {
"description": "",
"content": {
"application/json": {
"text/plain": {
"schema": {
"type": "string"
}
@@ -5137,7 +5173,7 @@
}
},
"info": {
"title": "Maestro - feat/new-customer-monitoring",
"title": "Maestro - SOLENG-4628-feat/restrict-asset-options",
"description": "This is the Maestro API",
"version": "1.0.0",
"contact": {}
@@ -5413,6 +5449,9 @@
"ConnectionToCatalogDto": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
@@ -7872,13 +7911,35 @@
"tags"
]
},
"CustomerLink": {
"type": "object",
"properties": {
"href": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"iconSrc": {
"type": "string"
}
},
"required": [
"href",
"name",
"description"
]
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"type": "string"
"$ref": "#/components/schemas/CustomerLink"
}
}
},
@@ -7886,13 +7947,13 @@
"links"
]
},
"CustomerDto": {
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"type": "string"
"$ref": "#/components/schemas/CustomerLink"
}
}
},
+2623 -9871
View File
File diff suppressed because it is too large Load Diff
+33 -32
View File
@@ -6,7 +6,7 @@
"private": true,
"license": "UNLICENSED",
"engines": {
"node": "18.10.0"
"node": "^18.17.0"
},
"scripts": {
"preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
@@ -27,30 +27,31 @@
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.33.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/cli": "^9.4.2",
"@nestjs/common": "^9.4.0",
"@nestjs/config": "^2.3.1",
"@nestjs/core": "^9.4.0",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.33.1-beta.2",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
"@nestjs/common": "^9.4.3",
"@nestjs/config": "^2.3.4",
"@nestjs/core": "^9.4.3",
"@nestjs/mapped-types": "^1.2.2",
"@nestjs/microservices": "^9.4.0",
"@nestjs/microservices": "^9.4.3",
"@nestjs/passport": "^9.0.3",
"@nestjs/platform-express": "^9.4.0",
"@nestjs/schematics": "^9.1.0",
"@nestjs/platform-express": "^9.4.3",
"@nestjs/schematics": "^9.2.0",
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cron-parser": "^4.4.0",
"cron-parser": "^4.9.0",
"dotenv": "^14.3.2",
"elastic-apm-node": "^3.36.0",
"helmet": "^5.1.0",
"jsonwebtoken": "^9.0.0",
"elastic-apm-node": "^3.50.0",
"helmet": "^5.1.1",
"jsonwebtoken": "^9.0.2",
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
@@ -60,39 +61,39 @@
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"protospack": "2.5.2",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.4.0"
"swagger-ui-express": "^4.6.3"
},
"overrides": {
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/express": "^4.17.13",
"@types/express": "^4.17.17",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.8",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
"@types/multer": "^1.4.7",
"@types/node": "^16.11.41",
"@types/node": "^16.18.52",
"@types/passport-facebook": "^2.1.11",
"@types/passport-google-oauth20": "^2.0.11",
"@types/passport-oauth2": "^1.4.11",
"@types/passport-oauth2": "^1.4.12",
"@types/supertest": "^2.0.12",
"@typescript-eslint/eslint-plugin": "^5.62.0",
"@typescript-eslint/parser": "^5.62.0",
"eslint": "^8.18.0",
"eslint-config-prettier": "^8.5.0",
"eslint": "^8.49.0",
"eslint-config-prettier": "^8.10.0",
"eslint-plugin-prettier": "^4.2.1",
"jest": "^27.5.1",
"nock": "^13.2.7",
"prettier": "^2.7.1",
"nock": "^13.3.3",
"prettier": "^2.8.8",
"source-map-support": "^0.5.20",
"supertest": "^6.2.3",
"supertest": "^6.3.3",
"ts-jest": "^27.1.5",
"ts-loader": "^9.3.1",
"ts-node": "^10.8.1",
"tsconfig-paths": "^3.14.1",
"typescript": "^4.6.3"
"ts-loader": "^9.4.4",
"ts-node": "^10.9.1",
"tsconfig-paths": "^3.14.2",
"typescript": "^4.9.5"
}
}
@@ -24,6 +24,7 @@ import {
ConnectionDetailsRes,
ConnectionRes,
ConnectionsRes,
GetAllConnectionsReq,
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
@@ -138,7 +139,7 @@ export class ConnectionController {
async getAllConnections(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Query() queries,
@Query() queries: GetAllConnectionsReq,
): Promise<ConnectionsRes> {
this.logger.info('/connections - Get All Connection');
+14 -2
View File
@@ -9,6 +9,7 @@ import {
ConnectionToCatalog,
} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities';
import {
GetAllConnectionRequest,
GetAllConnectionResponse,
GetConnectionDetailsResponse,
GetConnectionResponse,
@@ -21,7 +22,7 @@ export type ConnectionCredentialsType =
| 'oauth'
| 'api_key'
| 'service_account'
| 'headers_authF';
| 'headers_auth';
export interface ConnectionApiConnection {
config_id: string;
plugin: string;
@@ -94,7 +95,6 @@ export class ConnectionDto implements Connection {
export class ConnectionToCatalogDto implements ConnectionToCatalog {
// ---Automatically generated information - will not be sent by the frontend--- //
id: string;
customer_id: string;
updated_at: string;
created_at: string;
@@ -104,6 +104,9 @@ export class ConnectionToCatalogDto implements ConnectionToCatalog {
customer_name: string;
// ---Information sent by the frontend--- //
@ApiPropertyOptional()
id: string;
@ApiProperty()
name: string;
@@ -147,6 +150,15 @@ export class UpdateConnectionDto extends PickType(ConnectionDto, [
'properties',
]) {}
export class GetAllConnectionsReq implements GetAllConnectionRequest {
@ApiPropertyOptional()
search?: string;
@ApiPropertyOptional()
page?: string;
@ApiPropertyOptional()
size?: string;
}
export class ConnectionsRes implements GetAllConnectionResponse {
@ApiProperty({ type: [ConnectionToCatalogDto] })
connections: ConnectionToCatalogDto[];
+15 -10
View File
@@ -4,27 +4,28 @@ import {
Body,
Controller,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Put,
Query,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { ApiOkResponse, ApiProduces, ApiTags } from '@nestjs/swagger';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { CustomersService } from './customers.service';
import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@ApiInternalOnlyController()
@ApiTags('Customers')
@Controller('customers')
@Authenticated()
@@ -41,7 +42,8 @@ export class CustomersController {
}
@Get(':id/links')
async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> {
@ApiOkResponse({ type: CustomerLinksResponse })
async getCustomerLinks(@Param('id') id: string) {
this.logger.info('getCustomerLinks', { id });
const links = await this.customersService.getLinks(id);
return { links };
@@ -49,17 +51,20 @@ export class CustomersController {
@Put(':id/links')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
setCustomerLinks(
@Body() body: CustomerDto,
@Param('id') id,
): Promise<IdResponse> {
@ApiOkResponse()
@HttpCode(HttpStatus.OK)
async setCustomerLinks(
@Body() body: CustomerLinkRequest,
@Param('id') id: string,
) {
const { links } = body;
this.logger.info('setCustomerLinks', { id, links });
return this.customersService.setLinks(id, links);
await this.customersService.setLinks(id, links);
}
@Get('token')
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
@ApiProduces('text/plain')
async getCustomerToken(
@Query('exp') exp: StringValue,
@User() user: RequestUser,
+15 -5
View File
@@ -1,12 +1,22 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty } from '@nestjs/swagger';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class CustomerDto {
export class CustomerLink implements Link {
@ApiProperty()
links: Link[];
href: string;
@ApiProperty()
name: string;
@ApiProperty()
description: string;
@ApiPropertyOptional()
iconSrc: string;
}
export class CustomerLinkRequest {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
}
export class CustomerLinksResponse {
@ApiProperty()
links: Link[];
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
}
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
interface Values {
jdbc_user: string;
+6 -2
View File
@@ -1,6 +1,9 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { PipelineServicesNames, PipelinesServiceInterface } from 'protospack';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces';
@@ -11,9 +14,10 @@ import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;
+1 -1
View File
@@ -3,7 +3,7 @@ import {
GrpcOptions,
Transport,
} from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from 'protospack';
import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
import { credentials } from '@grpc/grpc-js';
const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+1 -1
View File
@@ -1,5 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from 'protospack/dist/lib/interfaces';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export class IPipelineV2 {
@ApiProperty()
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces';
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreateTransformationsRequest {
transformations: Transformation[];