Compare commits

...
25 Commits
Author SHA1 Message Date
Gabriel Amorim 35f35ddeb3 Merge pull request #222 from dadosfera/SOLENG-4628-feat/restrict-asset-options
[SOLENG-4628] feat/restrict asset options
2024-07-10 12:03:32 -03:00
Gabriel Rosa 90155a8811 FIX: cahnged self-hosted actions runner to run test 2024-07-10 10:43:40 -03:00
Gabriel Rosa ae8baca694 Merge branch 'main' into SOLENG-4628-feat/restrict-asset-options 2024-07-08 09:39:53 -03:00
Gabriel Amorim 82b8751f65 Merge pull request #221 from dadosfera/fix/docker-compose
FIX: use docker compose instead of docker-compose
2024-06-28 16:58:18 -03:00
Gabriel Rosa 78ee64fc45 FIX: login aws on test 2024-06-28 16:55:31 -03:00
Gabriel Rosa 2f30837fed FIX: use docker compose instead of docker-compose 2024-06-28 16:47:45 -03:00
Gabriel Amorim f46b1b62db Merge pull request #220 from dadosfera/feat/new-customer-monitoring
FEAT: new customer monitoring dashboard
2024-06-28 16:37:38 -03:00
Gabriel Rosa 8b6cd8a88e FEAT: customer monitoring dashboard 2024-06-21 09:21:27 -03:00
Gabriel Rosa 0c333b476a FEAT: new customer monitoring 2024-06-20 10:50:07 -03:00
Gabriel Rosa 1aa9371245 FIX: Preserve slash escape 2024-05-08 18:09:56 -03:00
Gabriel Rosa 14fcf4a0a1 CI: format the node_exporter url with format() 2024-05-08 18:01:44 -03:00
Gabriel Rosa 24f824bcd1 FIX: update protospack version 2024-05-08 17:17:55 -03:00
Gabriel Rosa fcdd0bf571 Merge branch 'main' into SOLENG-4628-feat/restrict-asset-options 2024-05-08 15:37:45 -03:00
Gabriel Amorim c416a2cb97 Merge pull request #219 from dadosfera/feat/pipeline-monitoring-dashboard
Feat: pipeline monitoring dashboard
2024-05-06 16:52:26 -03:00
Gabriel Rosa d03021fc9d new protospack release 2024-05-06 15:28:26 -03:00
Gabriel Rosa 574cf48726 FIX: npm audit fix 2024-05-02 12:11:46 -03:00
Gabriel Rosa 7cef404acf FEAT: get monitoring dashboard url 2024-05-02 12:11:35 -03:00
Gabriel Rosa 3c32c3c7d4 CI: Update actions/checkout to v4 2024-04-22 14:57:01 -03:00
Gabriel Amorim d44531b963 Merge pull request #218 from dadosfera/feat/choose-deploy-dockerhub
[ TECN-6603 ] CI: update action versions and choose wether to upload to dockerhub
2024-01-10 10:05:47 -03:00
Gabriel Rosa 36c80b480e CI: update action versions and choose wether to upload to dockerhub 2024-01-04 14:58:47 -03:00
Rafael Santana eeef97679c Merge pull request #217 from dadosfera/ci/dockerhub
UPDATE: updating maestro to deploy image to dockerhub
2023-11-26 21:18:32 -03:00
Gabriel Amorim 4b20388d45 DOCS: update some routes swagger 2023-09-20 09:39:43 -03:00
Gabriel Amorim 33fd2bd35f FIX: update old protospack library 2023-09-20 09:39:27 -03:00
Gabriel Amorim 654c837d3f FEAT: new node version 2023-09-18 11:34:23 -03:00
Gabriel Amorim 416b58080d CI: fixed conventional-changelog-eslint version 2023-09-18 11:33:09 -03:00
27 changed files with 4361 additions and 10598 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ module.exports = {
extends: [ extends: [
'eslint:recommended', 'eslint:recommended',
'plugin:@typescript-eslint/recommended', 'plugin:@typescript-eslint/recommended',
'plugin:prettier/recommended', 'prettier',
], ],
root: true, root: true,
env: { env: {
+20 -18
View File
@@ -13,6 +13,11 @@ on:
- stg - stg
- stg2 - stg2
- prd - prd
push_to_dockerhub:
description: "Push image to Dockerhub?"
required: true
type: boolean
default: false
jobs: jobs:
extract_environment: extract_environment:
@@ -39,15 +44,15 @@ jobs:
new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }} new_release_version: ${{ (steps.semantic.outputs.new_release_published == 'true' && steps.semantic.outputs.new_release_version) || (github.event_name == 'workflow_dispatch' && '0.0.0') }}
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v4
- if: github.event_name != 'workflow_dispatch' - if: github.event_name != 'workflow_dispatch'
name: Semantic Release name: Semantic Release
uses: cycjimmy/semantic-release-action@v3 uses: cycjimmy/semantic-release-action@v4
id: semantic id: semantic
with: with:
extra_plugins: | extra_plugins: |
conventional-changelog-eslint@4.0.0 conventional-changelog-eslint@4
branches: | branches: |
[ [
'main' 'main'
@@ -76,16 +81,12 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v4
- name: Update Pip - name: Update Pip
run: | run: |
python3 -m pip install --upgrade pip python3 -m pip install --upgrade pip
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI - name: Install AWS CLI
run: | run: |
python3 -m pip install awscli --upgrade python3 -m pip install awscli --upgrade
@@ -95,14 +96,14 @@ jobs:
python3 -m pip install awsebcli --upgrade python3 -m pip install awsebcli --upgrade
- name: Configure AWS Region - name: Configure AWS Region
uses: aws-actions/configure-aws-credentials@v1-node16 uses: aws-actions/configure-aws-credentials@v4
id: aws id: aws
with: with:
aws-region: us-east-1 aws-region: us-east-1
- name: Login to AWS ECR - name: Login to AWS ECR
id: login_ecr id: login_ecr
uses: aws-actions/amazon-ecr-login@v1 uses: aws-actions/amazon-ecr-login@v2
- name: Build, Tag, and Push Image to AWS ECR - name: Build, Tag, and Push Image to AWS ECR
env: env:
@@ -110,31 +111,32 @@ jobs:
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: | run: |
docker-compose -f build.docker-compose.yml build docker compose -f build.docker-compose.yml build
docker-compose -f build.docker-compose.yml push docker compose -f build.docker-compose.yml push
- name: Login to Docker Hub - name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
uses: docker/login-action@v2 uses: docker/login-action@v2
with: with:
username: dadosfera username: dadosfera
password: ${{ secrets.DOCKERHUB_PASSWORD }} password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build, Tag, and Push Image to Dockerhub - name: Build, Tag, and Push Image to Dockerhub
if: ${{inputs.push_to_dockerhub}}
env: env:
ENV: ${{ needs.extract_environment.outputs.environment }} ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: | run: |
docker-compose -f build.docker-compose.dockerhub.yml build docker compose -f build.docker-compose.dockerhub.yml build
docker-compose -f build.docker-compose.dockerhub.yml push docker compose -f build.docker-compose.dockerhub.yml push
- name: Create ZIP file to Deploy AWS Beanstalk - name: Create ZIP file to Deploy AWS Beanstalk
env: env:
ENV: ${{ needs.extract_environment.outputs.environment }} ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}} NODE_EXPORTER_URL: ${{format('{0}.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest', steps.aws.outputs.aws-account-id)}} # ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
run: | run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
@@ -162,7 +164,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v4
- name: Extract Docs BlockId and PageId - name: Extract Docs BlockId and PageId
env: env:
+8 -2
View File
@@ -6,10 +6,16 @@ on:
jobs: jobs:
test: test:
runs-on: self-hosted runs-on: [self-hosted, prd]
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v3 uses: actions/checkout@v4
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@v4
id: aws
with:
aws-region: us-east-1
- name: Build - name: Build
env: env:
+1 -1
View File
@@ -1 +1 @@
18.10.0 18.17
+1
View File
@@ -12,6 +12,7 @@
"start:debug" "start:debug"
], ],
"runtimeExecutable": "npm", "runtimeExecutable": "npm",
"runtimeVersion": "18.17",
"skipFiles": [ "skipFiles": [
"<node_internals>/**" "<node_internals>/**"
], ],
+3 -3
View File
@@ -1,5 +1,5 @@
# install all dependencies # install all dependencies
FROM node:18.10-alpine as packages FROM node:18.17-alpine as packages
WORKDIR /app WORKDIR /app
COPY package.json package-lock.json ./ COPY package.json package-lock.json ./
RUN apk add --no-cache aws-cli \ RUN apk add --no-cache aws-cli \
@@ -7,12 +7,12 @@ RUN apk add --no-cache aws-cli \
&& npm install && npm install
# unit test specific build # unit test specific build
FROM node:18.10-alpine as test FROM node:18.17-alpine as test
WORKDIR /app WORKDIR /app
COPY --from=packages /app/node_modules ./node_modules COPY --from=packages /app/node_modules ./node_modules
COPY . ./ COPY . ./
FROM node:18.10-alpine FROM node:18.17-alpine
WORKDIR /app WORKDIR /app
COPY . /app/ COPY . /app/
COPY --from=packages /app/node_modules ./node_modules COPY --from=packages /app/node_modules ./node_modules
+1 -1
View File
@@ -12,7 +12,7 @@ These instructions will allow you to get a working copy of the project on your l
### 📋 Requirements ### 📋 Requirements
- [NodeJS v18.10.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions) - [NodeJS v18.17 / NPM v9.6.7](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle) - Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
- Create your Access Key on the "Security credentials" menu - Create your Access Key on the "Security credentials" menu
- Set the Access Key on your local development machine - Set the Access Key on your local development machine
+231 -2
View File
@@ -133,7 +133,32 @@
}, },
"get": { "get": {
"operationId": "ConnectionController_getAllConnections", "operationId": "ConnectionController_getAllConnections",
"parameters": [], "parameters": [
{
"name": "search",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": { "responses": {
"200": { "200": {
"description": "", "description": "",
@@ -490,6 +515,45 @@
] ]
} }
}, },
"/pipelinesV2/monitoring-dashboard": {
"get": {
"operationId": "PipelinesController_getMonitoringDashboard",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
}
]
}
},
"/pipelinesV2": { "/pipelinesV2": {
"post": { "post": {
"operationId": "PipelinesController_create", "operationId": "PipelinesController_create",
@@ -1736,6 +1800,118 @@
] ]
} }
}, },
"/customers/{id}/links": {
"get": {
"operationId": "CustomersController_getCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinksResponse"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
}
]
},
"put": {
"operationId": "CustomersController_setCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinkRequest"
}
}
}
},
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/customers/token": {
"get": {
"operationId": "CustomersController_getCustomerToken",
"parameters": [
{
"name": "exp",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/health": { "/health": {
"get": { "get": {
"operationId": "HealthController_check", "operationId": "HealthController_check",
@@ -1752,7 +1928,7 @@
} }
}, },
"info": { "info": {
"title": "Maestro - feat/generate-token", "title": "Maestro - feat/new-customer-monitoring",
"description": "This is the Maestro API", "description": "This is the Maestro API",
"version": "1.0.0", "version": "1.0.0",
"contact": {} "contact": {}
@@ -2033,6 +2209,9 @@
"ConnectionToCatalogDto": { "ConnectionToCatalogDto": {
"type": "object", "type": "object",
"properties": { "properties": {
"id": {
"type": "string"
},
"name": { "name": {
"type": "string" "type": "string"
}, },
@@ -3124,6 +3303,56 @@
"updated_at", "updated_at",
"updated_by" "updated_by"
] ]
},
"CustomerLink": {
"type": "object",
"properties": {
"href": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"iconSrc": {
"type": "string"
}
},
"required": [
"href",
"name",
"description"
]
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
} }
} }
} }
+139 -17
View File
@@ -582,7 +582,32 @@
}, },
"get": { "get": {
"operationId": "ConnectionController_getAllConnections", "operationId": "ConnectionController_getAllConnections",
"parameters": [], "parameters": [
{
"name": "search",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": { "responses": {
"200": { "200": {
"description": "", "description": "",
@@ -2663,6 +2688,45 @@
] ]
} }
}, },
"/pipelinesV2/monitoring-dashboard": {
"get": {
"operationId": "PipelinesController_getMonitoringDashboard",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
}
]
}
},
"/pipelinesV2": { "/pipelinesV2": {
"post": { "post": {
"operationId": "PipelinesController_create", "operationId": "PipelinesController_create",
@@ -4962,7 +5026,16 @@
"/customers/{id}/links": { "/customers/{id}/links": {
"get": { "get": {
"operationId": "CustomersController_getCustomerLinks", "operationId": "CustomersController_getCustomerLinks",
"parameters": [], "parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": { "responses": {
"200": { "200": {
"description": "", "description": "",
@@ -4986,27 +5059,29 @@
}, },
"put": { "put": {
"operationId": "CustomersController_setCustomerLinks", "operationId": "CustomersController_setCustomerLinks",
"parameters": [], "parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"required": true, "required": true,
"content": { "content": {
"application/json": { "application/json": {
"schema": { "schema": {
"$ref": "#/components/schemas/CustomerDto" "$ref": "#/components/schemas/CustomerLinkRequest"
} }
} }
} }
}, },
"responses": { "responses": {
"200": { "200": {
"description": "", "description": ""
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
} }
}, },
"tags": [ "tags": [
@@ -5039,7 +5114,7 @@
"200": { "200": {
"description": "", "description": "",
"content": { "content": {
"application/json": { "text/plain": {
"schema": { "schema": {
"type": "string" "type": "string"
} }
@@ -5060,6 +5135,28 @@
] ]
} }
}, },
"/customers/monitoring-dashboard": {
"get": {
"operationId": "CustomersController_getCustomerMonitoringDashboard",
"parameters": [],
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/health": { "/health": {
"get": { "get": {
"operationId": "HealthController_check", "operationId": "HealthController_check",
@@ -5076,7 +5173,7 @@
} }
}, },
"info": { "info": {
"title": "Maestro - feat/generate-token", "title": "Maestro - SOLENG-4628-feat/restrict-asset-options",
"description": "This is the Maestro API", "description": "This is the Maestro API",
"version": "1.0.0", "version": "1.0.0",
"contact": {} "contact": {}
@@ -5352,6 +5449,9 @@
"ConnectionToCatalogDto": { "ConnectionToCatalogDto": {
"type": "object", "type": "object",
"properties": { "properties": {
"id": {
"type": "string"
},
"name": { "name": {
"type": "string" "type": "string"
}, },
@@ -7811,13 +7911,35 @@
"tags" "tags"
] ]
}, },
"CustomerLink": {
"type": "object",
"properties": {
"href": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"iconSrc": {
"type": "string"
}
},
"required": [
"href",
"name",
"description"
]
},
"CustomerLinksResponse": { "CustomerLinksResponse": {
"type": "object", "type": "object",
"properties": { "properties": {
"links": { "links": {
"type": "array", "type": "array",
"items": { "items": {
"type": "string" "$ref": "#/components/schemas/CustomerLink"
} }
} }
}, },
@@ -7825,13 +7947,13 @@
"links" "links"
] ]
}, },
"CustomerDto": { "CustomerLinkRequest": {
"type": "object", "type": "object",
"properties": { "properties": {
"links": { "links": {
"type": "array", "type": "array",
"items": { "items": {
"type": "string" "$ref": "#/components/schemas/CustomerLink"
} }
} }
}, },
+3757 -10479
View File
File diff suppressed because it is too large Load Diff
+35 -35
View File
@@ -6,7 +6,7 @@
"private": true, "private": true,
"license": "UNLICENSED", "license": "UNLICENSED",
"engines": { "engines": {
"node": "18.10.0" "node": "^18.17.0"
}, },
"scripts": { "scripts": {
"preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1", "preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
@@ -27,30 +27,31 @@
"test:e2e": "jest --config ./test/jest-e2e.json" "test:e2e": "jest --config ./test/jest-e2e.json"
}, },
"dependencies": { "dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0", "@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.31.0", "@dadosfera/protospack": "2.5.3",
"@grpc/grpc-js": "^1.6.7", "@dadosfera/protospack-v2": "3.33.1-beta.2",
"@grpc/proto-loader": "^0.6.13", "@grpc/grpc-js": "^1.9.3",
"@nestjs/cli": "^9.4.2", "@grpc/proto-loader": "^0.7.9",
"@nestjs/common": "^9.4.0", "@nestjs/cli": "^9.5.0",
"@nestjs/config": "^2.3.1", "@nestjs/common": "^9.4.3",
"@nestjs/core": "^9.4.0", "@nestjs/config": "^2.3.4",
"@nestjs/core": "^9.4.3",
"@nestjs/mapped-types": "^1.2.2", "@nestjs/mapped-types": "^1.2.2",
"@nestjs/microservices": "^9.4.0", "@nestjs/microservices": "^9.4.3",
"@nestjs/passport": "^9.0.3", "@nestjs/passport": "^9.0.3",
"@nestjs/platform-express": "^9.4.0", "@nestjs/platform-express": "^9.4.3",
"@nestjs/schematics": "^9.1.0", "@nestjs/schematics": "^9.2.0",
"@nestjs/swagger": "^6.3.0", "@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.0", "@nestjs/testing": "^9.4.3",
"axios": "^0.27.2", "axios": "^0.27.2",
"class-transformer": "^0.5.1", "class-transformer": "^0.5.1",
"class-validator": "^0.14.0", "class-validator": "^0.14.0",
"cron-parser": "^4.4.0", "cron-parser": "^4.9.0",
"dotenv": "^14.3.2", "dotenv": "^14.3.2",
"elastic-apm-node": "^3.36.0", "elastic-apm-node": "^3.50.0",
"helmet": "^5.1.0", "helmet": "^5.1.1",
"jsonwebtoken": "^9.0.0", "jsonwebtoken": "^9.0.2",
"jwk-to-pem": "^2.0.5", "jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0", "mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1", "ms": "^3.0.0-canary.1",
@@ -60,40 +61,39 @@
"passport-google-oauth20": "^2.0.0", "passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3", "passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0", "passport-mailchimp": "^1.1.0",
"protospack": "2.5.2",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"rxjs": "^7.5.5", "rxjs": "^7.5.5",
"swagger-ui-express": "^4.4.0" "swagger-ui-express": "^4.6.3"
}, },
"overrides": { "overrides": {
"multer": "1.4.5-lts.1" "multer": "1.4.5-lts.1"
}, },
"devDependencies": { "devDependencies": {
"@types/express": "^4.17.13", "@types/express": "^4.17.17",
"@types/jest": "27.0.2", "@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.8", "@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1", "@types/jwk-to-pem": "^2.0.1",
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
"@types/node": "^16.11.41", "@types/node": "^16.18.52",
"@types/passport-facebook": "^2.1.11", "@types/passport-facebook": "^2.1.11",
"@types/passport-google-oauth20": "^2.0.11", "@types/passport-google-oauth20": "^2.0.11",
"@types/passport-oauth2": "^1.4.11", "@types/passport-oauth2": "^1.4.12",
"@types/supertest": "^2.0.12", "@types/supertest": "^2.0.12",
"@typescript-eslint/eslint-plugin": "^5.29.0", "@typescript-eslint/eslint-plugin": "^5.62.0",
"@typescript-eslint/parser": "^5.29.0", "@typescript-eslint/parser": "^5.62.0",
"eslint": "^8.18.0", "eslint": "^8.49.0",
"eslint-config-prettier": "^8.5.0", "eslint-config-prettier": "^8.10.0",
"eslint-plugin-prettier": "^4.0.0", "eslint-plugin-prettier": "^4.2.1",
"jest": "^27.5.1", "jest": "^27.5.1",
"nock": "^13.2.7", "nock": "^13.3.3",
"prettier": "^2.7.1", "prettier": "^2.8.8",
"source-map-support": "^0.5.20", "source-map-support": "^0.5.20",
"supertest": "^6.2.3", "supertest": "^6.3.3",
"ts-jest": "^27.1.5", "ts-jest": "^27.1.5",
"ts-loader": "^9.3.1", "ts-loader": "^9.4.4",
"ts-node": "^10.8.1", "ts-node": "^10.9.1",
"tsconfig-paths": "^3.14.1", "tsconfig-paths": "^3.14.2",
"typescript": "^4.6.3" "typescript": "^4.9.5"
} }
} }
+19 -11
View File
@@ -67,7 +67,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
PIPELINE: { PIPELINE: {
title: { title: {
'pt-br': 'Coletar | Pipelines', 'pt-br': 'Coletar | Pipelines',
@@ -117,7 +116,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
CONNECTION: { CONNECTION: {
title: { title: {
'pt-br': 'Coletar | Fontes de dados', 'pt-br': 'Coletar | Fontes de dados',
@@ -157,7 +155,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
NETWORK_CONFIG: { NETWORK_CONFIG: {
title: { title: {
'pt-br': 'Coletar | Redes', 'pt-br': 'Coletar | Redes',
@@ -187,7 +184,6 @@ export const PERMISSIONS_GROUPS = {
// }, // },
}, },
}, },
OUTPUT: { OUTPUT: {
title: { title: {
'pt-br': 'Output', 'pt-br': 'Output',
@@ -237,7 +233,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
TRANSFORMATIONS: { TRANSFORMATIONS: {
title: { title: {
'pt-br': 'Micro-transformações', 'pt-br': 'Micro-transformações',
@@ -287,7 +282,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
CATALOG: { CATALOG: {
title: { title: {
'pt-br': 'Explorar | Catálogo', 'pt-br': 'Explorar | Catálogo',
@@ -368,7 +362,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
CONNECTORS: { CONNECTORS: {
title: { title: {
'pt-br': 'Conectores', 'pt-br': 'Conectores',
@@ -418,7 +411,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
SNOWFLAKE: { SNOWFLAKE: {
title: { title: {
'pt-br': 'Explorar | Consolidar', 'pt-br': 'Explorar | Consolidar',
@@ -438,7 +430,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
ZENDESK: { ZENDESK: {
title: { title: {
'pt-br': 'Zendesk', 'pt-br': 'Zendesk',
@@ -458,7 +449,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
DATAVIZ: { DATAVIZ: {
title: { title: {
'pt-br': 'Analisar | Visualização', 'pt-br': 'Analisar | Visualização',
@@ -497,7 +487,6 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
MODULES: { MODULES: {
title: { title: {
'pt-br': 'Módulos da Dadosfera', 'pt-br': 'Módulos da Dadosfera',
@@ -594,6 +583,25 @@ export const PERMISSIONS_GROUPS = {
}, },
}, },
}, },
CUSTOMER: {
title: {
'pt-br': 'Organização',
'en-us': 'Organization',
'es-es': 'Organización',
},
permissions: {
MONITORING_DASHBOARD: {
seqid: 47,
claim: 'customer:monitoring-dashboard',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Ver o dashboard de monitoramento',
'en-us': 'View the monitoring dashboard',
'es-es': 'Ver el dashboard de monitoreo',
},
},
},
},
}; };
export interface DadosferaModule { export interface DadosferaModule {
name: string; name: string;
+3 -1
View File
@@ -164,6 +164,8 @@ export class AuthClientService implements OnModuleInit {
async oauthSignIn(data: { username: string; token: string }) { async oauthSignIn(data: { username: string; token: string }) {
const { username, token } = data; const { username, token } = data;
return lastValueFrom(this.authService.AuthOauthSignIn({ token, username })); return lastValueFrom(
this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
);
} }
} }
@@ -24,6 +24,7 @@ import {
ConnectionDetailsRes, ConnectionDetailsRes,
ConnectionRes, ConnectionRes,
ConnectionsRes, ConnectionsRes,
GetAllConnectionsReq,
UpdateConnectionDto, UpdateConnectionDto,
} from './dtos/connection'; } from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection'; import { CreateConnectionDto } from './dtos/connection';
@@ -138,7 +139,7 @@ export class ConnectionController {
async getAllConnections( async getAllConnections(
@User() user: RequestUser, @User() user: RequestUser,
@Language() language: LanguageEnum, @Language() language: LanguageEnum,
@Query() queries, @Query() queries: GetAllConnectionsReq,
): Promise<ConnectionsRes> { ): Promise<ConnectionsRes> {
this.logger.info('/connections - Get All Connection'); this.logger.info('/connections - Get All Connection');
+14 -2
View File
@@ -9,6 +9,7 @@ import {
ConnectionToCatalog, ConnectionToCatalog,
} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities'; } from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities';
import { import {
GetAllConnectionRequest,
GetAllConnectionResponse, GetAllConnectionResponse,
GetConnectionDetailsResponse, GetConnectionDetailsResponse,
GetConnectionResponse, GetConnectionResponse,
@@ -21,7 +22,7 @@ export type ConnectionCredentialsType =
| 'oauth' | 'oauth'
| 'api_key' | 'api_key'
| 'service_account' | 'service_account'
| 'headers_authF'; | 'headers_auth';
export interface ConnectionApiConnection { export interface ConnectionApiConnection {
config_id: string; config_id: string;
plugin: string; plugin: string;
@@ -94,7 +95,6 @@ export class ConnectionDto implements Connection {
export class ConnectionToCatalogDto implements ConnectionToCatalog { export class ConnectionToCatalogDto implements ConnectionToCatalog {
// ---Automatically generated information - will not be sent by the frontend--- // // ---Automatically generated information - will not be sent by the frontend--- //
id: string;
customer_id: string; customer_id: string;
updated_at: string; updated_at: string;
created_at: string; created_at: string;
@@ -104,6 +104,9 @@ export class ConnectionToCatalogDto implements ConnectionToCatalog {
customer_name: string; customer_name: string;
// ---Information sent by the frontend--- // // ---Information sent by the frontend--- //
@ApiPropertyOptional()
id: string;
@ApiProperty() @ApiProperty()
name: string; name: string;
@@ -147,6 +150,15 @@ export class UpdateConnectionDto extends PickType(ConnectionDto, [
'properties', 'properties',
]) {} ]) {}
export class GetAllConnectionsReq implements GetAllConnectionRequest {
@ApiPropertyOptional()
search?: string;
@ApiPropertyOptional()
page?: string;
@ApiPropertyOptional()
size?: string;
}
export class ConnectionsRes implements GetAllConnectionResponse { export class ConnectionsRes implements GetAllConnectionResponse {
@ApiProperty({ type: [ConnectionToCatalogDto] }) @ApiProperty({ type: [ConnectionToCatalogDto] })
connections: ConnectionToCatalogDto[]; connections: ConnectionToCatalogDto[];
+29 -10
View File
@@ -4,26 +4,28 @@ import {
Body, Body,
Controller, Controller,
Get, Get,
HttpCode,
HttpStatus,
Inject, Inject,
Param, Param,
Put, Put,
Query, Query,
UseFilters, UseFilters,
} from '@nestjs/common'; } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger'; import { ApiOkResponse, ApiProduces, ApiTags } from '@nestjs/swagger';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { import {
Authenticated, Authenticated,
RequireAllPermissions, RequireAllPermissions,
} from 'src/decorators/authentication.decorator'; } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { CustomersService } from './customers.service'; import { CustomersService } from './customers.service';
import { CustomerDto, CustomerLinksResponse } from './dtos/customers'; import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator'; import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms'; import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@ApiInternalOnlyController()
@ApiTags('Customers') @ApiTags('Customers')
@Controller('customers') @Controller('customers')
@Authenticated() @Authenticated()
@@ -40,7 +42,8 @@ export class CustomersController {
} }
@Get(':id/links') @Get(':id/links')
async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> { @ApiOkResponse({ type: CustomerLinksResponse })
async getCustomerLinks(@Param('id') id: string) {
this.logger.info('getCustomerLinks', { id }); this.logger.info('getCustomerLinks', { id });
const links = await this.customersService.getLinks(id); const links = await this.customersService.getLinks(id);
return { links }; return { links };
@@ -48,17 +51,20 @@ export class CustomersController {
@Put(':id/links') @Put(':id/links')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
setCustomerLinks( @ApiOkResponse()
@Body() body: CustomerDto, @HttpCode(HttpStatus.OK)
@Param('id') id, async setCustomerLinks(
): Promise<IdResponse> { @Body() body: CustomerLinkRequest,
@Param('id') id: string,
) {
const { links } = body; const { links } = body;
this.logger.info('setCustomerLinks', { id, links }); this.logger.info('setCustomerLinks', { id, links });
return this.customersService.setLinks(id, links); await this.customersService.setLinks(id, links);
} }
@Get('token') @Get('token')
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN) @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
@ApiProduces('text/plain')
async getCustomerToken( async getCustomerToken(
@Query('exp') exp: StringValue, @Query('exp') exp: StringValue,
@User() user: RequestUser, @User() user: RequestUser,
@@ -71,4 +77,17 @@ export class CustomersController {
}; };
return this.customersService.generateToken(exp, data); return this.customersService.generateToken(exp, data);
} }
@Get('monitoring-dashboard')
@RequireAllPermissions(
PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
)
async getCustomerMonitoringDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getCustomerMonitoringDashboard');
const metadata = PackTheMetadata(user);
return this.customersService.getMonitoringDashboardUrl(metadata);
}
} }
+9 -2
View File
@@ -4,11 +4,18 @@ import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config'; import { DucClient } from '../duc/client.config';
import { CustomersController } from './customers.controller'; import { CustomersController } from './customers.controller';
import { CustomersService } from './customers.service'; import { CustomersService } from './customers.service';
import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
const client = new DucClient(); const ducClient = new DucClient();
const piFactoryClient = new PipelinesClientConfiguration();
@Module({ @Module({
imports: [ClientsModule.register([client.providerOptions])], imports: [
ClientsModule.register([
ducClient.providerOptions,
piFactoryClient.providerOptions,
]),
],
controllers: [CustomersController], controllers: [CustomersController],
providers: [CustomersService, DadosferaLogger], providers: [CustomersService, DadosferaLogger],
exports: [CustomersService], exports: [CustomersService],
@@ -22,20 +22,34 @@ import {
} from '@aws-sdk/client-secrets-manager'; } from '@aws-sdk/client-secrets-manager';
import getEnv from 'src/utils/getEnv'; import getEnv from 'src/utils/getEnv';
import { logger } from 'elastic-apm-node'; import { logger } from 'elastic-apm-node';
import {
ReadService,
ProtoServices as PipelineProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { Metadata } from '@grpc/grpc-js';
import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
// This function will accept any string, which may result in a bug. // This function will accept any string, which may result in a bug.
@Injectable() @Injectable()
export class CustomersService implements OnModuleInit { export class CustomersService implements OnModuleInit {
private customerService: CustomersProtoService; private customerService: CustomersProtoService;
private pipelineReadService: ReadService.PipelineV2ReadService;
constructor( constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc, @Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
@Inject(PipelinesClientConfiguration.name)
private readonly pipelinesGrpcClient: ClientGrpc,
) {} ) {}
onModuleInit() { onModuleInit() {
this.customerService = this.grpcClient.getService<CustomersProtoService>( this.customerService = this.grpcClient.getService<CustomersProtoService>(
ProtoServices.CustomersProtoService, ProtoServices.CustomersProtoService,
); );
this.pipelineReadService =
this.pipelinesGrpcClient.getService<ReadService.PipelineV2ReadService>(
PipelineProtoServices.PipelineV2ReadService,
);
} }
async getLinks(customerId: string) { async getLinks(customerId: string) {
@@ -120,4 +134,21 @@ export class CustomersService implements OnModuleInit {
// const decoded = jwt.decode(jwt_token, { complete: true }); // const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token; return jwt_token;
} }
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
const res = await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: '45',
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata,
),
);
logger.info('Done');
return res;
}
} }
+15 -5
View File
@@ -1,12 +1,22 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty } from '@nestjs/swagger'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class CustomerDto { export class CustomerLink implements Link {
@ApiProperty() @ApiProperty()
links: Link[]; href: string;
@ApiProperty()
name: string;
@ApiProperty()
description: string;
@ApiPropertyOptional()
iconSrc: string;
}
export class CustomerLinkRequest {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
} }
export class CustomerLinksResponse { export class CustomerLinksResponse {
@ApiProperty() @ApiProperty({ type: [CustomerLink] })
links: Link[]; links: CustomerLink[];
} }
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces'; import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
interface Values { interface Values {
jdbc_user: string; jdbc_user: string;
+6 -2
View File
@@ -1,6 +1,9 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common'; import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices'; import { ClientGrpc } from '@nestjs/microservices';
import { PipelineServicesNames, PipelinesServiceInterface } from 'protospack'; import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { lastValueFrom } from 'rxjs'; import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces'; import { IIdRequest } from './interfaces';
@@ -11,9 +14,10 @@ import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit { export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface; private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger; logger: DadosferaLogger;
constructor( constructor(
@Inject(DadosferaLogger) @Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger, dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name) @Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc, private readonly grpcClient: ClientGrpc,
) { ) {
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces'; import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreatePipelineDto { export interface ICreatePipelineDto {
input: IdRequest; input: IdRequest;
+1 -1
View File
@@ -3,7 +3,7 @@ import {
GrpcOptions, GrpcOptions,
Transport, Transport,
} from '@nestjs/microservices'; } from '@nestjs/microservices';
import { PipelinePackages, PipelineProtoFilePath } from 'protospack'; import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
import { credentials } from '@grpc/grpc-js'; import { credentials } from '@grpc/grpc-js';
const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0'); const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+1 -1
View File
@@ -1,5 +1,5 @@
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
import { Info } from 'protospack/dist/lib/interfaces'; import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export class IPipelineV2 { export class IPipelineV2 {
@ApiProperty() @ApiProperty()
@@ -14,6 +14,7 @@ import {
Patch, Patch,
HttpException, HttpException,
BadRequestException, BadRequestException,
CacheTTL,
} from '@nestjs/common'; } from '@nestjs/common';
import { import {
ApiCreatedResponse, ApiCreatedResponse,
@@ -86,6 +87,18 @@ export class PipelinesController {
this.logger = dadosferaLogger.logger; this.logger = dadosferaLogger.logger;
} }
@Get('monitoring-dashboard')
async getMonitoringDashboard(@User() user: RequestUser) {
this.logger.info('PipelinesController - getMonitoringDashboard', { user });
const metadata = PackTheMetadata(user);
const response =
await this.pipelinesClientService.getMonitoringDashboardUrl(metadata);
return response;
}
@Post() @Post()
@ApiCreatedResponse({ type: IPipelineV2 }) @ApiCreatedResponse({ type: IPipelineV2 })
async create( async create(
@@ -321,4 +321,22 @@ export class PipelinesService implements OnModuleInit {
); );
}); });
} }
async getMonitoringDashboardUrl(metadata: Metadata) {
this.logger.info('PipelinesClientService - getMonitoringDashboardUrl');
const res = await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: '83',
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata,
),
);
this.logger.info('Done');
return res;
}
} }
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from 'protospack/dist/lib/interfaces'; import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
export interface ICreateTransformationsRequest { export interface ICreateTransformationsRequest {
transformations: Transformation[]; transformations: Transformation[];