Compare commits

..
Author SHA1 Message Date
Gabriel Rosa b280871bce CI: fix syntax 2024-01-04 14:44:20 -03:00
Gabriel Rosa fd73663940 CI: fix syntax 2024-01-04 14:37:32 -03:00
Gabriel Rosa e420bf6547 CI: choose wether to deploy to dockerhub 2024-01-04 10:25:51 -03:00
Gabriel Rosa cc4cc99ccc FEAT: new refreshToken response
- temporarily disable mixpanel tracking
2023-12-09 11:58:03 -03:00
Gabriel Rosa fd0d68b149 FIX: removed state 2023-12-07 19:39:09 -03:00
Gabriel Rosa 98759b1cf5 FIX: logging oauth error 2023-12-07 17:27:30 -03:00
Gabriel Rosa 695fac97e9 FIX: get frontend url 2023-12-07 11:01:38 -03:00
Gabriel Rosa 86b076246e FEAT: magalu id login 2023-12-06 20:36:14 -03:00
Rafael Santana faa98cae8b UPDATE: triple equals 2023-12-01 13:20:40 -03:00
Rafael Santana 313ecc81ec UPDATE: missing semicolon 2023-12-01 13:20:04 -03:00
Rafael Santana 820d71d16d UPDATE: Adding the cloud_environment to point to the correct nimbus in maestro 2023-12-01 13:13:26 -03:00
Rafael Santana 734978b76f UPDATE: Increasing log level 2023-11-29 17:47:08 -03:00
133 changed files with 14603 additions and 15446 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ module.exports = {
extends: [
'eslint:recommended',
'plugin:@typescript-eslint/recommended',
'prettier',
'plugin:prettier/recommended',
],
root: true,
env: {
+126 -50
View File
@@ -3,7 +3,6 @@ on:
push:
branches:
- main
- beta
workflow_dispatch:
inputs:
environment:
@@ -12,7 +11,13 @@ on:
type: choice
options:
- stg
- stg2
- prd
push_to_dockerhub:
description: "Push image to Dockerhub?"
required: true
type: boolean
default: false
jobs:
extract_environment:
@@ -29,8 +34,6 @@ jobs:
echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT
elif [ ${GITHUB_REF} == "refs/heads/main" ]; then
echo "environment=prd" >> $GITHUB_OUTPUT
elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
echo "environment=stg" >> $GITHUB_OUTPUT
fi
id: extract_environment
@@ -45,40 +48,38 @@ jobs:
- if: github.event_name != 'workflow_dispatch'
name: Semantic Release
uses: cycjimmy/semantic-release-action@v3
uses: cycjimmy/semantic-release-action@v4
id: semantic
with:
extra_plugins: |
conventional-changelog-eslint@4.0.0
branches: |
[
'main',
{
name: 'alpha',
prerelease: true
},
{
name: 'beta',
prerelease: true
}
'main'
]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build_ecr_image:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
deploy-info:
if: ${{ github.event_name == 'workflow_dispatch'}}
needs: [extract_environment, semantic_release]
runs-on:
[self-hosted, "prd"]
runs-on: ubuntu-latest
steps:
- name: Printing stats
- name: Create summary
env:
EVENT: ${{ github.event_name }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ENV: ${{ needs.extract_environment.outputs.environment }}
run: echo ${GITHUB_REF#refs/heads/}
run: echo "### Deploy da branch \`$GITHUB_REF_NAME\` no ambiente **$ENV** :rocket:" >> $GITHUB_STEP_SUMMARY
deploy:
if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }}
needs: [extract_environment, semantic_release]
runs-on:
[self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
steps:
- name: Checkout
uses: actions/checkout@v4
@@ -86,6 +87,10 @@ jobs:
run: |
python3 -m pip install --upgrade pip
- name: Install Docker Compose
run: |
python3 -m pip install docker-compose --upgrade
- name: Install AWS CLI
run: |
python3 -m pip install awscli --upgrade
@@ -110,43 +115,114 @@ jobs:
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
docker-compose -f build.docker-compose.yml build
docker-compose -f build.docker-compose.yml push
# - name: Create ZIP file to Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
# IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
# ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
# NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
# run: |
# sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
# sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
# sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
# sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
# zip deploy.zip docker-compose.yml -r .ebextensions
- name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
uses: docker/login-action@v2
with:
username: dadosfera
password: ${{ secrets.DOCKERHUB_PASSWORD }}
# - name: Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
# AWS_REGION: us-east-1
# APP_NAME: ${{ github.event.repository.name }}
# run: |
# eb use $APP_NAME-$ENV
# echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
# eb deploy
- name: Build, Tag, and Push Image to Dockerhub
if: ${{inputs.push_to_dockerhub}}
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker-compose -f build.docker-compose.dockerhub.yml build
docker-compose -f build.docker-compose.dockerhub.yml push
- name: Create ZIP file to Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
run: |
sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
zip deploy.zip docker-compose.yml -r .ebextensions
- name: Deploy AWS Beanstalk
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
AWS_REGION: us-east-1
APP_NAME: ${{ github.event.repository.name }}
run: |
eb use $APP_NAME-$ENV
echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml
eb deploy
- name: Remove Docker's Trash
if: always()
run: |
docker system prune --volumes -a -f
docker system df
api_docs:
needs: [extract_environment, semantic_release, deploy]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
k8s-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: 'oracle'
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
- name: Extract Docs BlockId and PageId
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
shell: bash
run: |
if [ $ENV == "stg2" ]; then
echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
elif [ $ENV == "stg" ]; then
echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
elif [ $ENV == "prd" ]; then
echo "block_id=$PRD_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
echo "page_id=$PRD_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
fi
id: extract_docs_info
- name: Generate API docs
env:
DOCS_URL: ${{ secrets.DOCS_URL }}
DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }}
DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }}
EVENT: ${{ github.event_name }}
RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
run: |
if [ ${EVENT} != "workflow_dispatch" ]; then
sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json
fi
sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json
sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json
curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL
- name: Generate External API docs
env:
DOCS_URL: ${{ secrets.DOCS_URL }}
DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
DOCS_PAGE_ID: ${{secrets.EXTERNAL_DOCS_PAGE_ID}}
DOCS_BLOCK_ID: ${{secrets.EXTERNAL_DOCS_BLOCK_ID}}
EVENT: ${{ github.event_name }}
RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
run: |
if [ ${EVENT} != "workflow_dispatch" ]; then
sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.external.json
fi
sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.external.json
sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.external.json
sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.external.json
curl -X POST -H 'Content-Type: application/json' -d @docsfera.external.json $DOCS_URL
-137
View File
@@ -1,137 +0,0 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+11 -6
View File
@@ -6,18 +6,23 @@ on:
jobs:
test:
runs-on: [self-hosted, prd]
env:
APP_NAME: ${{ github.event.repository.name }}
runs-on: self-hosted
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v3
- name: Build
run: docker build -t ${APP_NAME}_teste --target test .
env:
APP_NAME: ${{ github.event.repository.name }}
run: |
docker build -t $APP_NAME --target test .
- name: Run Test
run: docker run ${APP_NAME}_teste
env:
ENV: test
APP_NAME: ${{ github.event.repository.name }}
run: |
docker run --rm --entrypoint="npm" $APP_NAME run test
- name: Remove Docker's Trash
if: always()
-102
View File
@@ -1,102 +0,0 @@
name: Validate K8S Modifications
on:
pull_request:
branches:
- main
- beta
jobs:
extract_environment:
runs-on: ubuntu-22.04
outputs:
environment: ${{ steps.extract_environment.outputs.environment }}
steps:
- name: Extract Environment
run: |
if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then
echo "environment=prd" >> $GITHUB_OUTPUT
elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then
echo "environment=stg" >> $GITHUB_OUTPUT
fi
id: extract_environment
helmfile-check:
env:
HOME: /home/runner
needs: [extract_environment]
environment: ${{ needs.extract_environment.outputs.environment }}
runs-on: [self-hosted, "prd-oracle"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Determine DNS_HOST based on environment
id: set_dns
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
if [ "$ENV" = "prd" ]; then
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
elif [ "$ENV" = "stg" ]; then
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
fi
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
+1 -1
View File
@@ -1 +1 @@
18.17
18.10.0
+2 -2
View File
@@ -12,7 +12,7 @@
"start:debug"
],
"runtimeExecutable": "npm",
"runtimeVersion": "18.17",
"runtimeVersion": "18.10.0",
"skipFiles": [
"<node_internals>/**"
],
@@ -20,4 +20,4 @@
"console": "integratedTerminal"
}
]
}
}
+16 -61
View File
@@ -1,66 +1,21 @@
FROM node:18.17-alpine AS base_image
FROM base_image AS build_base
# install all dependencies
FROM node:18.10-alpine as packages
WORKDIR /app
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache \
aws-cli \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
COPY package*.json ./
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# run aws cli without mounting secret, because CI already has AWS credentials
FROM build_base AS ci_image
RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
RUN npm ci
COPY . .
COPY package.json package-lock.json ./
RUN apk add --no-cache aws-cli \
&& aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \
&& npm install
# unit test specific build
FROM ci_image AS test
ENV DUC_URL=0.0.0.0:50051
ENTRYPOINT ["npm", "run", "test"]
# dev build
FROM build_base AS dev
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
# flag --build-from-source is required to force-build sqlite3
RUN npm ci
COPY . .
ENTRYPOINT npm run start:dev
FROM ci_image AS prod_build
RUN npm run build
FROM base_image
FROM node:18.10-alpine as test
WORKDIR /app
COPY --from=prod_build /app/dist ./dist
COPY --from=prod_build /app/node_modules ./node_modules
COPY --from=prod_build /app/package*.json ./
RUN apk update
# needed packages to build dependencies from source
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
COPY --from=packages /app/node_modules ./node_modules
COPY . ./
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT npm run start:prod
FROM node:18.10-alpine
WORKDIR /app
COPY . /app/
COPY --from=packages /app/node_modules ./node_modules
RUN npm run build
EXPOSE 3333
ENTRYPOINT npm run start
+84 -88
View File
@@ -2,57 +2,52 @@
<image src="./assets/maestro.svg" style="width:10rem">
</p>
# Maestro
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
```mermaid
graph TD;
Frontend<-->Maestro;
Maestro<-->duc;
Maestro<-->pi-factory;
Maestro<-->in-factory;
```
## 🚀 Starting
É uma API REST, desenvolvida em NodeJs utilizando o Framework [NestJs](https://docs.nestjs.com/).
These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
## 🚀 Iniciando
### 📋 Requirements
Estas instruções permitirão que você obtenha uma cópia funcional do projeto em sua máquina local para desenvolvimento e testes.
- [NodeJS v18.10.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
- Create your Access Key on the "Security credentials" menu
- Set the Access Key on your local development machine
- Request access to the dev, stg and prd VPNs
### 📋 Requisitos
### 🔧 Installation<a id="installation"></a>
- [NodeJS v18.17 / NPM v9.6.7](https://nodejs.org/pt-br/download/)
- Clone the repository
> Dica: Utilize [NVM](https://github.com/nvm-sh/nvm) para gerenciar facilmente as versões do node
- SSH
- Solicite acesso à conta de desenvolvimento do AWS Console para **todos os serviços necessários**
- Crie sua Access Key no menu "Security credentials"
- Defina a Access Key em sua máquina de desenvolvimento local
- Solicite acesso às VPNs de stg e prd
```
git clone git@github.com:dadosfera/maestro.git
```
### 🔧 Instalação<a id="installation"></a>
or
- Clone o repositório
- HTTPS
```
git clone https://github.com/dadosfera/maestro.git
```
```sh
git clone git@github.com:dadosfera/maestro.git
```
- Selecione a versão correta do node (opcional, apenas se estiver usando o [NVM](https://github.com/nvm-sh/nvm)):
- Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
```sh
nvm use
```
- Instale as dependências do projeto:
- Install the project dependencies:
```sh
npm i
```
- Configure as seguintes variáveis de ambiente:
- Setup the following enviroment variables:
```
ENV=
@@ -63,41 +58,42 @@ Estas instruções permitirão que você obtenha uma cópia funcional do projeto
SM_OAUTH_PATH=
```
- Inicie o servidor:
- Start the server:
```sh
# dev mode
npm run start:dev
# or in debug mode
npm run start:debug
```
> Se preferir, utilize o debbuger do VSCode apertando F5
The service should start successfully.
O serviço deve iniciar com sucesso.
## 📄 Documentation
## 📄 Documentação
NestJs makes it easy to document each route using decorators on all requests and responses properties. It automatically generates a swagger for given information and provides a route to access it http://localhost:3333/api.
For more info check the [official documentation](https://docs.nestjs.com/openapi/introduction).
O NestJs facilita a documentação de cada rota usando decoradores em todas as propriedades de solicitações e respostas. Ele gera automaticamente um swagger para as informações fornecidas e fornece uma rota para acessá-lo em http://localhost:3333/api. Para mais informações, consulte a [documentação oficial](https://docs.nestjs.com/openapi/introduction).
### - Multiple documentations
### - Documentações múltiplas
We are currently generating **2 different** documentations: Internal and External.
Atualmente, estamos gerando **2 documentações diferentes**: Interna e Externa.
All routes that have the decorator `@ApiInternalOnly()` will not be visible on the **External** API swagger.
Todas as rotas que têm o decorador `@ApiInternalOnly()` não serão visíveis no swagger da API **Externa**.
- When you start the application with `npm run start` it will serve and generate the swagger JSON of the **External** API
- When you start the application with `npm run start:internal` it will serve and generate the swagger JSON of the **Internal** API
- When you run `npm run docs` it will generate the swagger JSON of both **Internal** and **External** API, and save them to `docsfera.json` and `docsfera.external.json` respectively;
- Quando você inicia a aplicação com `npm run start:dev`, ela servirá e gerará o JSON do swagger da API **Interna**
It is important to run `npm run docs` before every deploy so we can always have the most updated docs published.
- Quando você executa `npm run docs`, ele gerará o JSON do swagger de ambas as APIs **Interna** e **Externa** e os salvará em `docsfera.json` e `docsfera.external.json`, respectivamente;
## Authentication decorators
> Link para documentação interna: https://dadosfera.github.io/docsfera
É importante executar `npm run docs` antes de cada implantação para que sempre tenhamos as documentações mais atualizadas publicadas.
## Decoradores de autenticação
O Maestro possui utilitários para facilitar a autenticação do usuário em todos os controladores e rotas. Os seguintes decoradores estão disponíveis:
Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
### `@Authenticated`
Se o usuário precisar estar autenticado para fazer uma solicitação, podemos usar o decorador `@Authenticated` no controlador ou rota, conforme necessário.
If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
```ts
import { Authenticated } from '../../authentication/authentication.decorator';
@@ -109,7 +105,7 @@ class FooController {
@Get('bar')
async getBar() {
this.logger.info('usuário está autenticado!');
this.logger.info('user is authenticated!');
return { authenticated: true };
}
@@ -126,14 +122,14 @@ class FooController {
@Get('bar')
@Authenticated()
async getBar() {
this.logger.info('usuário está autenticado!');
this.logger.info('user is authenticated!');
return { authenticated: true };
}
@Post('bar')
async postBar() {
this.logger.info('usuário NÃO está autenticado!');
this.logger.info('user is NOT authenticated!');
return { authenticated: false };
}
@@ -142,7 +138,7 @@ class FooController {
### `@RequireAllPermissions`
Este decorador exige que **todas as permissões** listadas sejam concedidas ao usuário solicitante.
This decorator requires that **all permissions** listed are granted to the requesting user.
```ts
import { RequireAllPermissions } from '../../authentication/authentication.decorator';
@@ -162,7 +158,7 @@ class FooController {
### `@RequireSomePermission`
No caso seguinte, é necessário que o usuário tenha **pelo menos uma** das permissões listadas.
In the following case, the user is required to have **at least one** listed permission.
```ts
import { RequireSomePermission } from '../../authentication/authentication.decorator';
@@ -182,18 +178,18 @@ class FooController {
### `@AuthenticateCondition`
Se for necessária uma verificação de autenticação mais complicada, podemos usar o decorador `@AuthenticateCondition` para defini-la. A função personalizada deve retornar `true` para autenticar a solicitação.
If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
No exemplo a seguir:
In the following example:
- todas as rotas no controlador `FooController` só podem ser solicitadas a partir do localhost
- `POST /foo/bar` só pode ser solicitado a partir do localhost **e** por usuários do cliente com id `111...eef`
- all routes on the `FooController` controller can only be requested from localhost
- `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
```ts
import { AuthenticateCondition } from '../../authentication/authentication.decorator';
@Controller('foo')
// permitir solicitações apenas do localhost
// allow requests only from localhost
@AuthenticateCondition((request: Request) => request.ip === '::ffff:127.0.0.1')
class FooController {
/* ... */
@@ -204,7 +200,7 @@ class FooController {
}
@Post('bar')
// permitir solicitações apenas de um cliente específico
// allow requests only from a specific customer
@AuthenticateCondition(
(request: Request, user: RequestUser) =>
user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef',
@@ -215,11 +211,11 @@ class FooController {
}
```
### Nota sobre decoradores de autenticação
### Note on authentication decorators
- O método antigo de autenticação colocava os dados do usuário no campo `request.body.info`, o que impõe certos problemas em relação ao corpo da solicitação, pois esses dados devem vir do frontend sem qualquer modificação pelo Maestro. Agora, esse uso está ⚠️ **DESCONTINUADO** ⚠️. Estamos trabalhando para migrar para o decorador de parâmetro `@User`. O método antigo está funcionando enquanto a migração está em andamento.
- The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
- Os decoradores de autenticação podem ser usados juntos e todos eles **devem** passar para que a solicitação seja autenticada, mas, no caso geral, você não precisa (_e não gostaria de..._) usar todos eles juntos, pois você pode codificar toda a lógica de autenticação no decorador `@AuthenticateCondition`.
- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (_and wouldn't like to..._) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
```ts
import {
@@ -255,17 +251,17 @@ class FooController {
}
```
- Se `@RequireAllPermissions` e `@RequireSomePermission` forem usados com **apenas uma única permissão**, eles apresentam o **exatamente mesmo comportamento**.
- If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
```ts
// mesmo comportamento
// same behavior
@RequireAllPermissions(Permissions.BAR.MANAGE)
@RequireSomePermission(Permissions.BAR.MANAGE)
```
## Decorador de parâmetro `@User`
## `@User` parameter decorator
Os dados do usuário solicitante podem ser obtidos usando o decorador de parâmetro @User, como no exemplo a seguir:
The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
```ts
import { User, RequestUser } from '../../authentication/user.decorator';
@@ -283,7 +279,7 @@ class FooController {
}
```
Se o usuário precisar estar logado, defina `required` como `true`, como por exemplo:
If the user is required to be logged in, set `required` to `true`, as you would want in the `change-password` operation:
```ts
import { User, RequestUser } from '../../authentication/user.decorator';
@@ -309,46 +305,46 @@ class UserController {
}
```
## 📦 Desenvolvimento
## 📦 Development
### ⌨️ Estilo de Codificação
### ⌨️ Coding Style
Por padrão, usamos [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) com configurações padrão.
By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
**Recomendamos usar o Visual Studio Code e instalar as extensões recomendadas para facilitar o processo de desenvolvimento.**
**We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
### Padrão de commits
### Commits pattern
Nosso pipeline de fluxo de trabalho segue as especificações de commits convencionais ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) para liberar versões conforme necessário.
Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
Formato: `<type>[optional scope]: <description>`
Format: `<type>[optional scope]: <description>`
Exemplo: `FIX: ensure Range headers adhere more closely to RFC 2616`
Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
### Convenção de nomenclatura de branchs
### Branching naming convention
- **Feature**: Quaisquer alterações de código para um novo módulo ou caso de uso devem ser feitas em uma branch de feature. Esta branch é criada com base na branch `main`. Quando todas as alterações estiverem concluídas, será necessário um Pull Request/Merge Request para colocar todas essas alterações de volta na branch `main`. Exemplos: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
- **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
**Recomenda-se usar todas as letras em minúsculas e hífen (-) para separar palavras, a menos que seja um nome ou ID de item específico. O sublinhado (\_) pode ser usado para separar o ID e a descrição.**
**It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (\_) could be used to separate the ID and description.**
- **Bug Fix**: Se as alterações de código feitas na branch de feature foram rejeitadas após um lançamento, sprint ou demo, quaisquer correções necessárias após isso devem ser feitas na branch de correção de bug. Exemplos: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
- **Hot Fix**: Se houver necessidade de corrigir um bloqueador, fazer um patch temporário, aplicar uma mudança crítica de framework ou configuração que deva ser tratada imediatamente, ela deve ser criada como um Hotfix. Exemplos: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
- **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
- **Experimental**: Uma branch para experimentar. Qualquer nova feature ou ideia que não faça parte de um lançamento ou sprint. Exemplo: `experimental/dark-theme-support`.
- **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
### Fazendo um Pull Request
### Making a Pull Request
1. Comite suas alterações
2. Abra o Pull Request no GitHub
3. Envie o link do Pull Request no grupo de chat do Google da Microsfera para revisão e possível aprovação
1. Commit your changes
2. Open the Pull Request on GitHub
3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
## 🛠️ Construído com
## 🛠️ Built with
Algumas tecnologias usadas neste projeto:
Some technologies used in this project:
- [NestJS](https://docs.nestjs.com) - Framework para construir aplicações NodeJS eficientes e escaláveis no lado do servidor
- [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
## ⚙️ Arquitetura de Back-end
## ⚙️ Back-end Architecture
A arquitetura pode ser encontrada neste [link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
+2 -1
View File
@@ -1,4 +1,5 @@
version: "3.8"
services:
maestro:
build: .
image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+2 -1
View File
@@ -1,4 +1,5 @@
version: "3.8"
services:
maestro:
build: .
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd:${IMAGE_TAG}
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
-24
View File
@@ -1,24 +0,0 @@
apiVersion: v2
name: maestro
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
-126
View File
@@ -1,126 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.app_name }}
namespace: applications
labels:
app: {{ .Values.app_name }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: {{ .Values.app_name }}
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: {{ .Values.app_name }}
spec:
imagePullSecrets:
- name: {{ .Values.imagePullSecrets }}
nodeSelector:
"beta.kubernetes.io/os": linux
{{- if .Values.affinity }}
affinity:
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
tolerations:
- key: "kubernetes.azure.com/scalesetpriority"
operator: "Equal"
value: "spot"
effect: "NoSchedule"
containers:
- name: maestro
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
ports:
- containerPort: {{ .Values.containerPort }}
{{- if .Values.resources }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
env:
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
- name: AWS_REGION
value: "us-east-1"
- name: BASE_HOST
value: "maestro_prd"
- name: BUCKET_CUSTOMER_CSV_ASSETS
value: {{ .Values.maestro.bucket_customer_csv_assets }}
- name: CONNECTORS_INDEX
value: "connectors"
- name: DUC_URL
value: {{ .Values.maestro.duc_url }}
- name: ELASTIC_APM_ENVIRONMENT
value: {{ .Values.maestro.env }}
- name: ELASTIC_APM_SERVER_URL
value: "https://apm-server.dadosfera.ai"
- name: ELASTIC_APM_SERVICE_NAME
value: {{ .Values.maestro.apm_service }}
- name: ENV
value: {{ .Values.maestro.env }}
- name: INFACTORY_URL
value: {{ .Values.maestro.in_factory_url }}
- name: LOGGER_GELF_HOST
value: "logstash-pipelines.dadosfera.ai"
- name: LOGGER_GELF_PORT
value: "{{ .Values.maestro.logger_gelf_port }}"
- name: LOGGER_CONSOLE_EXTRA
value: "true"
- name: NIMBUS_BASE_URL
value: "http://nimbus-api"
- name: NPM_TOKEN
value: {{ .Values.maestro.npm_token }}
- name: PB_TOKEN_PATH
value: {{ .Values.maestro.pb_token_path }}
- name: PIFACTORY_URL
value: {{ .Values.maestro.pi_factory_url }}
- name: SM_OAUTH_PATH
value: {{ .Values.maestro.sm_oauth_path }}
- name: TRFACTORY_URL
value: {{ .Values.maestro.tr_factory_url }}
- name: UPLOAD_FILE_AGENT_CONNECTION
value: {{ .Values.maestro.upload_file_agent_connection }}
- name: OPEN_CUSTOMER_ID
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
name: prd-duc
key: jwt_token
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: AWS_DEFAULT_REGION
@@ -1,32 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
pathType: Prefix
-36
View File
@@ -1,36 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
-40
View File
@@ -1,40 +0,0 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-{{ .Values.app_name }}
namespace: applications
labels:
app: {{ .Values.app_name }}
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-{{ .Values.app_name }}
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
remoteRef:
key: prd/microservices/aws_credentials/maestro
version: "AWSCURRENT"
property: AWS_ACCESS_KEY_ID
- secretKey: AWS_SECRET_ACCESS_KEY
remoteRef:
key: prd/microservices/aws_credentials/maestro
version: "AWSCURRENT"
property: AWS_SECRET_ACCESS_KEY
- secretKey: AWS_DEFAULT_REGION
remoteRef:
key: prd/microservices/aws_credentials/maestro
version: "AWSCURRENT"
property: AWS_DEFAULT_REGION
- secretKey: jwt_token
remoteRef:
key: {{ .Values.maestro.jwt_token_secret_id }}
version: "AWSCURRENT"
property: token
-18
View File
@@ -1,18 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.app_name }}
namespace: applications
labels:
app: {{ .Values.app_name }}
spec:
type: ClusterIP
ports:
- name: {{ .Values.app_name }}
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: {{ .Values.app_name }}
-16
View File
@@ -1,16 +0,0 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity: null
-67
View File
@@ -1,67 +0,0 @@
# Default values for metabase.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 3
hostname: maestro.dadosfera.ai
image:
repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
type: ClusterIP
port: 3333
targetPort: 3333
ingress:
enabled: false
port: 3333
resources:
requests:
cpu: 100m
memory: 1500Mi
limits:
cpu: 2000m
memory: 2Gi
maestro:
aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
duc_url: duc.dadosfera.ai
in_factory_url: in-factory.dadosfera.ai
bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153"
env: prd
apm_service: maestro
jwt_token_secret_id: prd/root/jwt_token
logger_gelf_port: "1026"
npm_token: npm_Xp17h3daMkORcZ55NY95Ez4gRfdzsQ3UvINP
pb_token_path: prd/root/productboard_token
pi_factory_url: pi-factory.dadosfera.ai
sm_oauth_path: prd/root/oauth_applications
tr_factory_url: in-factory.dadosfera.ai
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- product
-53
View File
@@ -1,53 +0,0 @@
releases:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro-unimed.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
# - name: maestro.restricted_ip
# value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,10.70.1.0/24"
-30
View File
@@ -1,30 +0,0 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
- name: private-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
set:
- name: app_name
value: maestro-private
- name: hostname
value: private-maestro.stg.dadosfera.ai
# Customer id
- name: maestro.dedicated_proxy
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
- name: replicaCount
value: 1
- name: affinity
value: null
- name: resources
value: null
- name: maestro.restricted_ip
value: "137.131.167.254/32,10.70.0.0/16,163.176.242.27/32"
@@ -1,3 +1,4 @@
version: "3.8"
services:
maestro:
image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
@@ -1,3 +1,4 @@
version: "3.8"
services:
maestro:
image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}
+2 -231
View File
@@ -133,32 +133,7 @@
},
"get": {
"operationId": "ConnectionController_getAllConnections",
"parameters": [
{
"name": "search",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "page",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
},
{
"name": "size",
"required": false,
"in": "query",
"schema": {
"type": "string"
}
}
],
"parameters": [],
"responses": {
"200": {
"description": "",
@@ -515,45 +490,6 @@
]
}
},
"/pipelinesV2/monitoring-dashboard": {
"get": {
"operationId": "PipelinesController_getMonitoringDashboard",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
}
},
"tags": [
"PipelinesV2"
],
"security": [
{
"access-token": []
}
]
}
},
"/pipelinesV2": {
"post": {
"operationId": "PipelinesController_create",
@@ -1800,118 +1736,6 @@
]
}
},
"/customers/{id}/links": {
"get": {
"operationId": "CustomersController_getCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinksResponse"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
}
]
},
"put": {
"operationId": "CustomersController_setCustomerLinks",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerLinkRequest"
}
}
}
},
"responses": {
"200": {
"description": ""
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/customers/token": {
"get": {
"operationId": "CustomersController_getCustomerToken",
"parameters": [
{
"name": "exp",
"required": true,
"in": "query",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"text/plain": {
"schema": {
"type": "string"
}
}
}
}
},
"tags": [
"Customers"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
]
}
},
"/health": {
"get": {
"operationId": "HealthController_check",
@@ -1928,7 +1752,7 @@
}
},
"info": {
"title": "Maestro - feat/new-customer-monitoring",
"title": "Maestro - feat/generate-token",
"description": "This is the Maestro API",
"version": "1.0.0",
"contact": {}
@@ -2209,9 +2033,6 @@
"ConnectionToCatalogDto": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"name": {
"type": "string"
},
@@ -3303,56 +3124,6 @@
"updated_at",
"updated_by"
]
},
"CustomerLink": {
"type": "object",
"properties": {
"href": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"iconSrc": {
"type": "string"
}
},
"required": [
"href",
"name",
"description"
]
},
"CustomerLinksResponse": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
},
"CustomerLinkRequest": {
"type": "object",
"properties": {
"links": {
"type": "array",
"items": {
"$ref": "#/components/schemas/CustomerLink"
}
}
},
"required": [
"links"
]
}
}
}
+312 -2082
View File
File diff suppressed because it is too large Load Diff
+1 -4
View File
@@ -4,6 +4,7 @@ declare global {
NODE_ENV: 'test';
ENV: 'local' | 'stg' | 'prd' | 'test';
LOCAL_ENV: 'stg' | 'prd';
CLOUD_ENVIRONMENT: 'aws' | 'gcp' | 'mgc';
DUC_URL: string;
INFACTORY_URL: string;
@@ -12,10 +13,6 @@ declare global {
INTERNAL_SWAGGER: 'true' | 'false';
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
}
}
}
-1
View File
@@ -4,7 +4,6 @@
"compilerOptions": {
"assets": [
"**/*.proto",
"assets/**/*",
{
"include": "i18n/**/*",
"watchAssets": true
+13602 -8078
View File
File diff suppressed because it is too large Load Diff
+39 -50
View File
@@ -6,10 +6,10 @@
"private": true,
"license": "UNLICENSED",
"engines": {
"node": "18.17"
"node": "18.10.0"
},
"scripts": {
"co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
"preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
"proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
"prebuild": "rimraf dist",
"build": "nest build",
@@ -20,91 +20,80 @@
"start:prod": "node dist/main",
"docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest --detectOpenHandles --forceExit",
"test": "jest",
"test:watch": "jest --watch",
"test:cov": "jest --coverage",
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
"@nestjs/common": "^9.4.3",
"@nestjs/config": "^2.3.4",
"@nestjs/core": "^9.4.3",
"@dadosfera/protospack-v2": "3.32.0-beta.2",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/cli": "^9.4.2",
"@nestjs/common": "^9.4.0",
"@nestjs/config": "^2.3.1",
"@nestjs/core": "^9.4.0",
"@nestjs/mapped-types": "^1.2.2",
"@nestjs/microservices": "^9.4.3",
"@nestjs/microservices": "^9.4.0",
"@nestjs/passport": "^9.0.3",
"@nestjs/platform-express": "^9.4.3",
"@nestjs/schematics": "^9.2.0",
"@nestjs/platform-express": "^9.4.0",
"@nestjs/schematics": "^9.1.0",
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"@nestjs/testing": "^9.4.0",
"axios": "^1.6.2",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"cron-parser": "^4.4.0",
"dotenv": "^14.3.2",
"elastic-apm-node": "^3.50.0",
"handlebars": "^4.7.8",
"helmet": "^5.1.1",
"jsonwebtoken": "^9.0.2",
"elastic-apm-node": "^3.36.0",
"helmet": "^5.1.0",
"jsonwebtoken": "^9.0.0",
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
"passport-google-oauth20": "^2.0.0",
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"protospack": "2.5.2",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.6.3"
"swagger-ui-express": "^4.4.0"
},
"overrides": {
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/express": "^4.17.13",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jsonwebtoken": "^8.5.8",
"@types/jwk-to-pem": "^2.0.1",
"@types/multer": "^1.4.12",
"@types/node": "^16.18.52",
"@types/multer": "^1.4.7",
"@types/node": "^16.11.41",
"@types/passport-facebook": "^2.1.11",
"@types/passport-google-oauth20": "^2.0.11",
"@types/passport-oauth2": "^1.4.12",
"@types/passport-oauth2": "^1.4.11",
"@types/supertest": "^2.0.12",
"@typescript-eslint/eslint-plugin": "^5.62.0",
"@typescript-eslint/parser": "^5.62.0",
"eslint": "^8.49.0",
"eslint-config-prettier": "^8.10.0",
"eslint-plugin-prettier": "^4.2.1",
"@typescript-eslint/eslint-plugin": "^5.35.0",
"@typescript-eslint/parser": "^5.35.0",
"eslint": "^8.18.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.0.0",
"jest": "^27.5.1",
"nock": "^13.3.3",
"prettier": "^2.8.8",
"nock": "^13.2.7",
"prettier": "^2.7.1",
"source-map-support": "^0.5.20",
"supertest": "^6.3.3",
"supertest": "^6.2.3",
"ts-jest": "^27.1.5",
"ts-loader": "^9.4.4",
"ts-node": "^10.9.1",
"tsconfig-paths": "^3.14.2",
"typescript": "^4.9.5"
"ts-loader": "^9.3.1",
"ts-node": "^10.8.1",
"tsconfig-paths": "^3.14.1",
"typescript": "^4.6.3"
}
}
+1 -16
View File
@@ -26,15 +26,9 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
import { ProductboardModule } from './modules/productboard/productboard.module';
import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
@Module({
controllers: [],
providers: [
DadosferaLogger,
{
@@ -64,15 +58,6 @@ import { ApiKeyModule } from './modules/api-key/api-key.module';
ProductboardModule,
MixpanelModule,
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
NetworkPolicyModule,
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
],
-130
View File
@@ -1,130 +0,0 @@
<!DOCTYPE html>
<html lang="pt-br">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Dadosfera Relatório de PII</title>
<link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
<style>
@page {
size: A4 landscape; /* Alterado para paisagem (landscape) */
margin: 15mm 10mm; /* Reduzido para proporcionar mais espaço */
}
body {
font-family: 'Quicksand', sans-serif;
color: #5c5c5c;
margin: 0;
padding: 10px;
font-size: 12px; /* Reduzindo o tamanho da fonte */
}
.container {
margin: 0;
width: 100%;
}
.header {
display: flex;
align-items: center;
margin-bottom: 20px;
}
.logo {
max-width: 150px; /* Reduzida para economizar espaço */
height: auto;
}
h1 {
color: #0d003b;
font-weight: 700;
margin-left: 20px;
font-size: 24px; /* Tamanho ajustado */
}
table {
width: 100%;
border-collapse: collapse;
margin-top: 15px;
table-layout: fixed; /* Importante: define larguras fixas */
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
border: 1px solid #d0d0d0;
}
th {
background-color: #1700a2;
color: white;
font-weight: bold;
text-align: left;
padding: 8px 10px;
border: 1px solid #3a26b8;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
td {
padding: 6px 10px;
border: 1px solid #d0d0d0;
font-size: 11px; /* Tamanho ajustado */
word-wrap: break-word; /* Permite quebra de palavras */
overflow-wrap: break-word;
}
/* Definindo larguras específicas para cada coluna */
th:nth-child(1), td:nth-child(1) { width: 14%; } /* Database */
th:nth-child(2), td:nth-child(2) { width: 14%; } /* Schema */
th:nth-child(3), td:nth-child(3) { width: 17%; } /* Tabela */
th:nth-child(4), td:nth-child(4) { width: 17%; } /* Coluna */
th:nth-child(5), td:nth-child(5) { width: 13%; } /* Tipo de Dado */
th:nth-child(6), td:nth-child(6) { width: 25%; } /* Regras PII */
tr:nth-child(even) {
background-color: #f9f9f9;
}
tr:nth-child(odd) {
background-color: white;
}
.info-section {
margin-top: 20px;
color: #5c5c5c;
}
.timestamp {
font-style: italic;
text-align: right;
margin-top: 15px;
font-size: 0.9em;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<img src="https://dadosfera.ai/wp-content/webp-express/webp-images/uploads/2022/06/Logo-Dadosfera1-1.png.webp" alt="Logo Dadosfera" class="logo">
<h1>Relatório de PII</h1>
</div>
<div class="info-section">
<p>Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.</p>
</div>
<table>
<thead>
<tr>
<th>Database</th>
<th>Schema</th>
<th>Tabela</th>
<th>Coluna</th>
<th>Tipo de Dado</th>
<th>Regras PII</th>
</tr>
</thead>
<tbody>
{{#each dados}}
<tr>
<td>{{database_name}}</td>
<td>{{table_schema}}</td>
<td>{{table_name}}</td>
<td>{{column_name}}</td>
<td>{{data_type}}</td>
<td>{{pii_rules}}</td>
</tr>
{{/each}}
</tbody>
</table>
<p class="timestamp">Gerado em: {{dataGeracao}}</p>
</div>
</body>
</html>
+12 -20
View File
@@ -17,7 +17,6 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -100,7 +99,6 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -122,12 +120,6 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -448,18 +440,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
+9 -41
View File
@@ -4,7 +4,6 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -18,7 +17,6 @@ import {
import { RequestUser } from '../decorators/user.decorator';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
@Injectable()
export class AuthenticationGuard
@@ -33,7 +31,6 @@ export class AuthenticationGuard
dadosferaLogger: DadosferaLogger,
private reflector: Reflector,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService
) {
this.pems = new Map();
this.logger = dadosferaLogger.logger;
@@ -51,40 +48,20 @@ export class AuthenticationGuard
});
}
async canActivate(ctx: ExecutionContext): Promise<boolean> {
canActivate(ctx: ExecutionContext): boolean {
const authFunctions = this.reflector.getAllAndMerge<
AuthenticationFunction[]
>(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
const mustBeAuthenticated = authFunctions.length > 0;
const request = ctx.switchToHttp().getRequest();
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!mustBeAuthenticated) {
// no need to be authenticated
return true;
}
const request = ctx.switchToHttp().getRequest();
const apiKey = request.get('X-api-key');
if (apiKey) {
const {
api_key
} = await this.apiKeyService.get(apiKey);
request.user = {
user_id: api_key.user_id,
username: api_key.username,
permissions: api_key.permissions,
customer_id: api_key.customer_id,
customer_name: api_key.customer_name,
customer_tier: api_key.customer_tier,
customer_modules: api_key.customer_modules,
access_token: apiKey,
};
return true;
}
const accessToken = this.validateToken(request, mustBeAuthenticated);
if (!accessToken) {
// couldn't load valid token
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
@@ -102,9 +79,13 @@ export class AuthenticationGuard
request,
mustBeAuthenticated: boolean,
): RequestUser | false {
const accessToken = request.get('Authorization');
const accessToken: string = request.get('Authorization');
let accessTokenPayload: RequestUser;
this.logger.info(`mustBeAuthenticated: ${mustBeAuthenticated}`);
this.logger.info(
`accessToken (first 5 chars): ${accessToken?.slice(0, 5)}`,
);
// If the user isn't authenticated, an error will occurr anywhere here.
// Fancy error avoidance isn't performed by purpose, such as avoiding to access null values.
try {
@@ -136,18 +117,6 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
@@ -156,7 +125,6 @@ export class AuthenticationGuard
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
// TODO: for backwards compatibility. remove in the future
+21 -48
View File
@@ -67,6 +67,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
PIPELINE: {
title: {
'pt-br': 'Coletar | Pipelines',
@@ -116,6 +117,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
CONNECTION: {
title: {
'pt-br': 'Coletar | Fontes de dados',
@@ -155,6 +157,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
NETWORK_CONFIG: {
title: {
'pt-br': 'Coletar | Redes',
@@ -184,6 +187,7 @@ export const PERMISSIONS_GROUPS = {
// },
},
},
OUTPUT: {
title: {
'pt-br': 'Output',
@@ -233,6 +237,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
TRANSFORMATIONS: {
title: {
'pt-br': 'Micro-transformações',
@@ -282,6 +287,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
CATALOG: {
title: {
'pt-br': 'Explorar | Catálogo',
@@ -340,6 +346,16 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
@@ -352,25 +368,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -420,6 +418,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
SNOWFLAKE: {
title: {
'pt-br': 'Explorar | Consolidar',
@@ -439,6 +438,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
ZENDESK: {
title: {
'pt-br': 'Zendesk',
@@ -458,6 +458,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
DATAVIZ: {
title: {
'pt-br': 'Analisar | Visualização',
@@ -496,6 +497,7 @@ export const PERMISSIONS_GROUPS = {
},
},
},
MODULES: {
title: {
'pt-br': 'Módulos da Dadosfera',
@@ -592,25 +594,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
CUSTOMER: {
title: {
'pt-br': 'Organização',
'en-us': 'Organization',
'es-es': 'Organización',
},
permissions: {
MONITORING_DASHBOARD: {
seqid: 47,
claim: 'customer:monitoring-dashboard',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Ver o dashboard de monitoramento',
'en-us': 'View the monitoring dashboard',
'es-es': 'Ver el dashboard de monitoreo',
},
},
},
},
};
export interface DadosferaModule {
name: string;
@@ -618,16 +601,6 @@ export interface DadosferaModule {
key: string;
permissionSeqId: number;
}
export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
{
name: 'Intelligence Module',
@@ -25,14 +25,6 @@ export function RequireSomePermission(
);
}
export function RequireModule(
key: string
) {
return createAuthenticatedDecorator((_, user: RequestUser) =>
user.customer_modules.some(module => module === key),
);
}
export function AuthenticateCondition(func: AuthenticationFunction) {
return createAuthenticatedDecorator(func);
}
+1 -9
View File
@@ -9,7 +9,6 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -53,7 +52,6 @@ describe('user.decorator', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -76,12 +74,6 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -183,5 +175,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
// UserTest(token);
UserTest(token);
});
-1
View File
@@ -11,7 +11,6 @@ export interface RequestUser {
customer_name: string;
customer_tier: string;
access_token: string;
customer_modules: string[];
}
export const User: (options?: { required?: boolean }) => ParameterDecorator =
+1 -20
View File
@@ -3,14 +3,11 @@ import { NestFactory } from '@nestjs/core';
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
import helmet from 'helmet';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { json, urlencoded } from 'express';
import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -21,29 +18,13 @@ async function bootstrap() {
const app = await NestFactory.create(AppModule, {
logger,
cors: {
origin: [
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
],
origin: '*',
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true,
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use(
'/catalog/register-dataset',
urlencoded({ extended: true, limit: '10mb' }),
);
}
configureSwagger(app);
await app.listen(3333);
if (process.env.KILL_AFTER_START) await app.close();
-68
View File
@@ -1,68 +0,0 @@
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@Controller('api-key')
@Authenticated()
@ApiTags('ApiKey')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
export class ApiKeyController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private readonly apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@ApiResponse({ type: CreateApiKeyResponseDto })
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
this.logger.info('POST /api-key', {
permissions: createApiKeyDto.permissions,
method: 'create'
});
const result = await this.apiKeyService.create(createApiKeyDto, user);
this.logger.info('POST /api-key success', {
id: result.id,
method: 'create'
});
return result;
}
@Get()
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
this.logger.info('GET /api-key', {
method: 'findAll'
});
const result = await this.apiKeyService.findAll(user);
this.logger.info('GET /api-key success', {
count: result.length,
method: 'findAll'
});
return result;
}
@Delete(':id')
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
this.logger.info('DELETE /api-key/:id', {
id,
method: 'remove'
});
await this.apiKeyService.remove(id, user);
this.logger.info('DELETE /api-key/:id success', {
id,
method: 'remove'
});
}
}
-18
View File
@@ -1,18 +0,0 @@
import { Module } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { ApiKeyController } from './api-key.controller';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([ducClient.providerOptions])
],
controllers: [ApiKeyController],
providers: [ApiKeyService, DadosferaLogger],
exports: [ApiKeyService]
})
export class ApiKeyModule {}
-50
View File
@@ -1,50 +0,0 @@
import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class ApiKeyService implements OnModuleInit {
private apiKeyService: ApiKeyWriteProtoService;
constructor(
@Inject(DucClient.name) private readonly client: ClientGrpc,
) {}
onModuleInit() {
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
}
create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
const metadata = PackTheMetadata(user);
return lastValueFrom(this.apiKeyService.CreateApiKey({
permissions: createApiKeyDto.permissions
}, metadata));
}
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
const metadata = PackTheMetadata(user);
console.log(metadata)
const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
return data.api_keys;
}
async remove(id: string, user: RequestUser) {
const metadata = PackTheMetadata(user);
await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
}
async get(key: string) {
const metadata = PackTheMetadata({});
return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
}
}
-39
View File
@@ -1,39 +0,0 @@
import { ApiProperty } from '@nestjs/swagger';
import { IsArray, IsNumber } from 'class-validator';
export class PermissionDto {
@ApiProperty({ type: Number })
id: number;
@ApiProperty({ type: String })
name: string;
}
export class ApiKeyBaseResponseDto {
@ApiProperty({ type: String, format: 'uuid' })
id: string;
@ApiProperty({ type: String })
key_mask: string;
@ApiProperty({ type: [PermissionDto] })
permissions: PermissionDto[];
@ApiProperty({ type: String, format: 'date-time' })
created_at: string;
@ApiProperty({ type: String })
created_by: string;
}
export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
@ApiProperty({ type: String })
key: string;
}
export class CreateApiKeyDto {
@ApiProperty({ type: [Number], description: 'Array of permission IDs' })
@IsArray()
@IsNumber({}, { each: true })
permissions: number[];
}
-33
View File
@@ -1,33 +0,0 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('assign')
@Authenticated()
export class AssignController {
constructor(private readonly assignService: AssignService) {}
@Put('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
}
@Get('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
}
}
-15
View File
@@ -1,15 +0,0 @@
import { Module } from '@nestjs/common';
import { AssignService } from './assign.service';
import { AssignController } from './assign.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AssignController],
providers: [AssignService, DadosferaLogger]
})
export class AssignModule {}
-38
View File
@@ -1,38 +0,0 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Metadata } from '@grpc/grpc-js';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from 'src/modules/duc/client.config';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
@Injectable()
export class AssignService implements OnModuleInit {
ducService: AssingProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<AssingProtoService>(
ProtoServices.AssingProtoService,
);
}
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
return data;
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -1,3 +0,0 @@
export class CreateAssignDto {
publicKey: string;
}
+110 -305
View File
@@ -12,7 +12,6 @@ import {
Redirect,
Req,
Param,
Res,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -27,7 +26,7 @@ import {
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest
AuthVerifyTotpMfaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -44,23 +43,16 @@ import {
AuthRefreshAccessTokenRes,
AuthSignInReq,
AuthSignInRes,
BulkEditRequest,
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request, Response } from 'express';
import { Request } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt, { JwtPayload } from 'jsonwebtoken';
import jwt from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
import { getFrontendUrl } from 'src/utils/getFrontendBaseUrl';
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -68,7 +60,7 @@ type CookiesValues = {
@Controller('auth')
export class AuthController {
logger: DadosferaLogger;
redirectUrl: string;
frontendRedirectUrl: string;
constructor(
@Inject(DadosferaLogger)
@@ -76,17 +68,7 @@ export class AuthController {
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
switch (process.env.ENV) {
case 'stg':
this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/auth/login`;
break;
case 'prd':
this.redirectUrl = `https://app.dadosfera.ai/auth/login`;
break;
default:
this.redirectUrl = `http://localhost:4200/auth/login`;
}
this.frontendRedirectUrl = getFrontendUrl('/auth/login').href;
}
@Post('sign-in')
@@ -94,66 +76,10 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
throw error;
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
return this.authClient.signIn({ username, password, totp }, metadata);
}
@Post('refresh-access-token')
@@ -162,26 +88,16 @@ export class AuthController {
async refreshAccessToken(
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const { refreshToken, customerName: customer_name } = body;
const metadata = PackTheMetadata({
customer_name,
language,
custom_host: frontHost,
});
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
return res.send(data);
this.logger.info(`metadata: ${metadata}`);
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
}
@ApiInternalOnlyEndpoint()
@@ -252,23 +168,16 @@ export class AuthController {
@ApiInternalOnlyEndpoint()
@Post('confirm-reset-password')
@HttpCode(HttpStatus.OK)
async confirmResetPassword(
@Body() body: AuthConfirmResetPasswordRequest,
@Headers('origin') origin: string,
) {
async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
this.logger.info('/auth - confirm-reset-password');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ custom_host: frontHost });
const { username, code, newPassword } = body;
return this.authClient.confirmResetPassword(
{
username,
code,
newPassword,
},
metadata,
);
return this.authClient.confirmResetPassword({
username,
code,
newPassword,
});
}
@ApiInternalOnlyEndpoint()
@@ -351,7 +260,13 @@ export class AuthController {
@UseGuards(AuthGuard('google-login'))
@Redirect()
async googleOauthCallback(@Req() req) {
const { url, email, token, language = 'pt-br' } = await this.callback(req);
const {
url,
email,
token,
refreshToken,
language = 'pt-br',
} = await this.callback(req);
if (url.searchParams.get('error')) {
this.logger.error('/oauth/google - ERROR');
return { url: url.href };
@@ -361,6 +276,7 @@ export class AuthController {
.oauthSignIn({
username: email,
token,
refreshToken,
})
.then(({ session }) => {
this.logger.info('/oauth/google - SUCESS');
@@ -390,18 +306,79 @@ export class AuthController {
return { url: url.href };
}
async callback(req: Request) {
@ApiInternalOnlyEndpoint()
@Get('oauth/magalu-id')
@UseGuards(AuthGuard('magalu-id-login'))
magaluIdOauth() {
this.logger.info('/oauth/magalu-id');
return true;
}
@ApiInternalOnlyEndpoint()
@Get('oauth/magalu-id/callback')
@UseGuards(AuthGuard('magalu-id-login'))
@Redirect()
async magaluIdOauthCallback(@Req() req) {
const {
url,
email,
token,
refreshToken,
language = 'pt-br',
} = await this.callback(req, 'magalu-id');
if (url.searchParams.get('error')) {
this.logger.error('/oauth/magalu-id/callback - ERROR');
this.logger.error(url.searchParams.get('error'));
return { url: url.href };
}
await this.authClient
.oauthSignIn({
username: email,
token,
refreshToken,
})
.then(({ session }) => {
this.logger.info('/oauth/magalu-id/callback - SUCESS');
url.searchParams.set('session', session);
})
.catch((err) => {
this.logger.error('/oauth/magalu-id/callback - LOGIN ERROR');
let error = OauthErrors.INVALID_CREDENTIALS[language].error;
let error_description =
OauthErrors.INVALID_CREDENTIALS[language].error_description;
switch (err.details) {
case ErrorCodes.USER.NOT_FOUND:
error = OauthErrors.USER_NOT_FOUND[language].error;
error_description =
OauthErrors.USER_NOT_FOUND[language].error_description(email);
break;
case ErrorCodes.AUTH.UNAUTHORIZED:
error = OauthErrors.INVALID_SESSION[language].error;
error_description =
OauthErrors.INVALID_SESSION[language].error_description;
break;
}
url.searchParams.set('error', error);
url.searchParams.set('error_description', error_description);
return null;
});
return { url: url.href };
}
async callback(req: Request, oauth_type?: 'google' | 'magalu-id') {
const { error, state } = req.query;
const { authInfo } = req;
const url = new URL(this.redirectUrl);
let email, token, error_title, error_description;
let language: 'pt-br' | 'en-us' = 'pt-br';
const stateObject = jwt.verify(
state as string,
process.env.JWT_PRIVATE_KEY,
);
if (typeof stateObject != 'string') language = stateObject.language;
const url = new URL(this.frontendRedirectUrl);
let email, token, refreshToken, error_title, error_description;
const language: 'pt-br' | 'en-us' = 'pt-br';
// let stateObject;
// try {
// stateObject = jwt.verify(state as string, process.env.JWT_PRIVATE_KEY);
// } catch (error) {
// console.log('error', error);
// }
// if (typeof stateObject != 'string') language = stateObject.language;
if (error || !authInfo) {
this.logger.error(error);
@@ -412,11 +389,16 @@ export class AuthController {
OauthErrors.INVALID_CREDENTIALS[language].error_description;
if (error) error_description += ` - [${error}]`;
} else {
const { accessToken } = authInfo as any;
const { _json: userInfo } = req.user as any;
const { accessToken, refreshToken: rt } = authInfo as any;
const { _json: userInfo = {} } = req.user as any;
email = userInfo.email;
token = accessToken;
refreshToken = rt;
if (oauth_type === 'magalu-id') {
const jwtDecoded = jwt.decode(token, { json: true });
email = jwtDecoded.email;
}
}
if (error_title) {
@@ -424,183 +406,6 @@ export class AuthController {
url.searchParams.set('error_description', error_description);
}
return { token, email, url, language };
}
@ApiInternalOnlyEndpoint()
@Post('users/block')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async blockUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest,
) {
this.logger.info('blockUsers - Starting request');
try {
const metadata = PackTheMetadata(user);
this.logger.debug('Calling blockUsers service', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language'),
},
});
const result = await this.authClient.blockUsers(body.users, metadata);
this.logger.info('blockUsers - Success', { result });
return result;
} catch (error) {
this.logger.error('blockUsers - Error', {
error: error.message,
stack: error.stack,
});
throw error;
}
}
@ApiInternalOnlyEndpoint()
@Post('users/unblock')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async unblockUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest,
) {
this.logger.info('unblockUsers');
const metadata = PackTheMetadata(user);
return this.authClient.unblockUsers(body.users, metadata);
}
@ApiInternalOnlyEndpoint()
@Post('users/reset')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async resetUsers(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() body: BulkEditRequest,
) {
this.logger.info('resetUsers');
const metadata = PackTheMetadata(user);
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
return { token, refreshToken, email, url, language };
}
}
+2
View File
@@ -8,6 +8,7 @@ import { AuthClientService } from './auth.service';
import { DucClient } from '../duc/client.config';
import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
import { getOauthSecrets } from 'src/utils/OauthSecrets';
import { MagaluIdStrategy } from './passport-strategies/magalu-id-strategy';
const client = new DucClient();
@Module({
@@ -17,6 +18,7 @@ const client = new DucClient();
AuthClientService,
DadosferaLogger,
GoogleLoginStrategy,
MagaluIdStrategy,
{ provide: 'OAUTH_SECRETS', useValue: getOauthSecrets() },
],
exports: [AuthClientService],
+20 -137
View File
@@ -1,10 +1,10 @@
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -17,22 +17,16 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
AuthOauthSignInRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -45,10 +39,6 @@ export class AuthClientService implements OnModuleInit {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async getPublicKeys() {
@@ -63,59 +53,25 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
}
async refreshAccessToken(
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
{ refreshToken }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
);
}
@@ -157,21 +113,19 @@ export class AuthClientService implements OnModuleInit {
);
}
async confirmResetPassword(
{ username, code, newPassword }: AuthConfirmResetPasswordRequest,
metadata: Metadata,
) {
async confirmResetPassword({
username,
code,
newPassword,
}: AuthConfirmResetPasswordRequest) {
this.logger.info('confirmResetPassword');
return lastValueFrom(
this.authService.AuthConfirmResetPassword(
{
username,
code,
newPassword,
},
metadata,
),
this.authService.AuthConfirmResetPassword({
username,
code,
newPassword,
}),
);
}
@@ -209,78 +163,7 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthGetSession({ session }));
}
async oauthSignIn(data: { username: string; token: string }) {
const { username, token } = data;
return lastValueFrom(
this.authService.AuthOauthSignIn({ token, username, refreshToken: '' }),
);
}
async blockUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('blockUsers - Service starting');
try {
this.logger.debug('Calling BlockUser gRPC method', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language'),
},
});
const response = await lastValueFrom<BulkEditResponse>(
this.authService.BlockUser({ users }, metadata),
);
this.logger.info('blockUsers - Service success', { response });
return response;
} catch (error) {
this.logger.error('blockUsers - Service error', {
error: error.message,
stack: error.stack,
});
throw error;
}
}
async unblockUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('unblockUsers');
return await lastValueFrom(
this.authService.UnblockUser({ users }, metadata),
);
}
async resetUsers(
users: string[],
metadata: Metadata,
): Promise<BulkEditResponse> {
this.logger.info('resetUsers');
try {
this.logger.debug('Calling ResetUser gRPC method', {
metadata: {
access_token: metadata.get('access_token'),
language: metadata.get('language'),
},
});
const response = await lastValueFrom<BulkEditResponse>(
this.authService.ResetUser({ users }, metadata),
);
this.logger.info('resetUsers - Success', { response });
return response;
} catch (error) {
this.logger.error('resetUsers - Error', {
error: error.message,
stack: error.stack,
});
throw error;
}
async oauthSignIn(data: AuthOauthSignInRequest) {
return lastValueFrom(this.authService.AuthOauthSignIn(data));
}
}
+3 -23
View File
@@ -67,28 +67,16 @@ export class AuthUser {
export class AuthCustomer {
@ApiProperty()
modules: string[];
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
displayName: string;
@ApiProperty()
tier: string;
@ApiProperty()
scheduleLimit: string;
@ApiProperty()
links: Link[];
@ApiProperty()
themeEnabled: boolean;
@ApiProperty()
enforceMfa: boolean;
}
export class AuthSignInReq implements AuthSignInRequest {
@@ -122,21 +110,13 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
refreshToken: string;
@ApiProperty()
userId: string;
customerName: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
permissions: string[];
@ApiProperty()
accessToken: string;
}
export interface BulkEditRequest {
users: string[];
}
export interface BulkEditResponse {
message: string;
successfulUsers: string[];
failedUsers: string[];
@ApiProperty()
refreshToken: string;
}
@@ -26,7 +26,6 @@ export class GoogleLoginStrategy extends PassportStrategy(
callbackURL: oauthSecrets['google-login'].redirect_uri,
scope: ['email', 'profile', 'openid'],
};
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
const verify = (
accessToken: string,
refreshToken: string,
@@ -0,0 +1,47 @@
import {
Strategy as Oauth2Strategy,
StrategyOptions,
VerifyCallback,
VerifyFunction,
} from 'passport-oauth2';
import { PassportStrategy } from '@nestjs/passport';
import { Inject, Injectable } from '@nestjs/common';
import { OauthSecrets } from 'src/utils/OauthSecrets';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
@Injectable()
export class MagaluIdStrategy extends PassportStrategy(
Oauth2Strategy,
'magalu-id-login',
) {
redirect_uri: string;
constructor() {
const options: StrategyOptions = {
clientID: process.env.MAGALU_ID_CLIENT_ID,
clientSecret: process.env.MAGALU_ID_CLIENT_SECRET,
callbackURL: process.env.MAGALU_ID_CALLBACK_URL,
scope: ['openid'],
authorizationURL: 'https://id.magalu.com/login',
tokenURL: 'https://id.magalu.com/oauth/token',
};
const verify: VerifyFunction = (
accessToken: string,
refreshToken: string,
profile: any,
verified: VerifyCallback,
) => {
return verified(null, profile, { accessToken, refreshToken });
};
super(options, verify);
}
// authenticate(req: Request, options: Record<string, any>) {
// const language =
// req.headers['dadosfera-lang'] || req.query.language || 'pt-br';
// // options.state = jwt.sign({ language }, process.env.JWT_PRIVATE_KEY);
// super.authenticate(req, options);
// console.log('authenticate', JSON.stringify(options));
// }
}
+1 -6
View File
@@ -5,11 +5,8 @@ import {
} from '@nestjs/microservices';
import { credentials } from '@grpc/grpc-js';
import { Catalog } from '@dadosfera/protospack-v2';
import { PlatformInterfaces } from '@dadosfera/protospack-v2';
const isLocalConnection =
process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
process.env.PIFACTORY_URL.includes('0.0.0.0');
const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
export class CatalogClientConfiguration {
public name = 'CatalogClientConfiguration';
@@ -20,13 +17,11 @@ export class CatalogClientConfiguration {
package: [
Catalog.ProtoPackages.ReadPackage,
Catalog.ProtoPackages.WritePackage,
PlatformInterfaces.ProtoPackages.WritePackage
],
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: [
Catalog.ProtoPaths.ReadFilePath,
Catalog.ProtoPaths.WriteFilePath,
PlatformInterfaces.ProtoPaths.WriteFilePath
],
loader: {
keepCase: true,
+4 -302
View File
@@ -13,9 +13,6 @@ import {
Put,
Query,
UseFilters,
HttpException,
HttpStatus,
Res,
} from '@nestjs/common';
import {
ApiCreatedResponse,
@@ -26,16 +23,14 @@ import {
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from '../../decorators/authentication.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
BatchRemoveRlsRulesRequest,
GetDatasetCatalogTaskRes,
ICatalogAllRequest,
ICatalogAllResponse,
@@ -54,14 +49,6 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
RegisterDatasetWithMetatadaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { Response } from 'express';
import { TypeParser } from 'src/utils/FileParser/parser-types';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -117,52 +104,6 @@ export class CatalogController {
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog/download - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
@@ -393,7 +334,7 @@ export class CatalogController {
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username, customer_modules } = user;
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
@@ -406,7 +347,6 @@ export class CatalogController {
user_id,
username,
language,
is_mask: customer_modules.some(mod => mod === 'pii')
});
const preview = await this.catalogService.getDatasetPreview(id, metadata);
@@ -693,242 +633,4 @@ export class CatalogController {
return response;
}
@Post('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async addRlsRule(@User() user: RequestUser, @Body() body: AddRlsRuleRequest) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.addRlsRule(body, metadata);
return response;
}
@Get('rls-rule/:id')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getOneRlsRule(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const idInt = parseInt(id);
const response = await this.catalogService.getOneRlsRule(idInt, metadata);
return response;
}
@Get('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getRlsRules(
@User() user: RequestUser,
@Query() query: GetRlsRulesRequest,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.getRlsRules(query, metadata);
return response;
}
@Delete('rls-rule/:id')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async removeRlsRule(@Param('id') id: string, @User() user: RequestUser) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const idInt = parseInt(id);
const response = await this.catalogService.removeRlsRule(idInt, metadata);
return response;
}
@Delete('rls-rule')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async batchRemoveRlsRules(
@Query() query: BatchRemoveRlsRulesRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
await this.catalogService.batchRemoveRlsRule(query, metadata);
return { message: 'OK' };
}
@Get('nimbus-dashboards')
@RequireAllPermissions(PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER)
async getNimbusDashboards(
@User() user: RequestUser,
@Body() body: GetNimbusDashboardsRequest,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const dashboards = await this.catalogService.getNimbusDashboards(
body,
metadata,
);
return JSON.parse(dashboards);
}
@Post('register-dataset')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async registerDatasetWithMetadataRequest(
@Body() body: RegisterDatasetWithMetatadaRequest,
@User() user: RequestUser,
) {
const { customer_id, customer_name, user_id, username } = user;
const logMetadata = {
customer_name: customer_name,
user_id: user_id,
method: 'POST',
path: '/catalog/register-dataset',
};
try {
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
this.logger.log(
`Request from user ${user_id} for customer ${customer_name}`,
logMetadata,
);
// Create table metadata
const tableMetadataBody = {
table_metadata: body.table_metadata,
info: {
customer: customer_name,
},
logMetadata: logMetadata,
};
const table_metadata_id = await this.catalogService.createTableMetadata(
tableMetadataBody,
);
this.logger.info(`table_metadata_id: ${table_metadata_id}`, logMetadata);
// Create column metadata
const columnMetadataBody = {
column_metadata: body.column_metadata,
info: {
customer: customer_name,
},
};
this.logger.info(
`Creating column metadata for table ${table_metadata_id}`,
logMetadata,
);
const column_metadata_ids =
await this.catalogService.createColumnMetadata(columnMetadataBody);
// Create data preview
const dataPreviewBody = {
data_preview: body.data_preview,
info: {
customer: customer_name,
},
};
this.logger.debug(
`Creating data preview for table ${table_metadata_id}`,
logMetadata,
);
const data_preview_id = await this.catalogService.createDataPreview(
dataPreviewBody,
);
// Catalog dataset item
this.logger.info(
`Cataloging dataset item for table ${table_metadata_id}`,
logMetadata,
);
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
this.logger.info(
`Dataset registration completed successfully for table ${table_metadata_id}`,
logMetadata,
);
return {
message: 'Dataset registered successfully',
table_metadata_id: table_metadata_id,
column_metadata_ids: column_metadata_ids,
data_preview_id: data_preview_id,
};
} catch (error) {
this.logger.error(
`Failed to register dataset. The following error occurred: ${error.response.data}`,
logMetadata,
);
throw new HttpException(
{
message: 'Ocorreu um erro ao registrar o dataset',
error: error.message,
code: 'REGISTRATION_FAILED',
details: error.message,
},
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
@Get('pii-reporter')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(
DADOSFERA_MODULES_KEYS.PII
)
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
this.logger.info('GET pii-reporter');
const metadata = PackTheMetadata(user);
try {
const {
file,
filename,
type
} = await this.catalogService.getPiiReporter(metadata, contentType);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', type);
// use res.end to send buffer
return res.end(file);
} catch (error) {
console.error(error)
this.logger.error(error.message);
}
}
}
}
+1 -6
View File
@@ -3,15 +3,12 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@@ -21,8 +18,6 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
ShareModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
+10 -264
View File
@@ -6,12 +6,6 @@ import {
Messages,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
Messages as PlatformInterfaceMessages,
WriteService as PlatformInterfaceWriteService,
ProtoServices as PlatformInterfacesProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PlatformInterfaces';
import {
BadRequestException,
HttpException,
HttpStatus,
Inject,
@@ -24,25 +18,11 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
import {
AddRlsRuleRequest,
GetNimbusDashboardsRequest,
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
catalogWriteService: WriteService.CatalogWriteServices;
platformWriteService: PlatformInterfaceWriteService.PlatformInterfacesWriteServices;
logger: any;
constructor(
@Inject(DadosferaLogger)
@@ -64,62 +44,27 @@ class CatalogService implements OnModuleInit {
this.grpcClient.getService<WriteService.CatalogWriteServices>(
ProtoServices.CatalogWriteServices,
);
this.platformWriteService =
this.grpcClient.getService<PlatformInterfaceWriteService.PlatformInterfacesWriteServices>(
PlatformInterfacesProtoServices.PlatformInterfacesWriteServices,
);
}
_getNimbusUrl(body) {
this.logger.debug(`Body: ${JSON.stringify(body)}`);
const customer = body.info.customer.toLowerCase();
if (process.env.ENV === 'prd') {
const cloud_environment = process.env.CLOUD_ENVIRONMENT || 'aws';
if (process.env.ENV === 'prd' && cloud_environment === 'aws') {
return `https://nimbus-${customer}.dadosfera.ai`;
}
if (process.env.ENV === 'prd' && cloud_environment === 'mgc') {
return `https://nimbus-${customer}.dadosfera.com`;
}
return `https://nimbus-${customer}.${process.env.ENV.replace(
'local',
'stg',
)}.dadosfera.ai`;
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type)
try {
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
const mimeTypes: Record<TypeParser, string> = {
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
return {
file,
filename: filename,
type: mimeTypes[type]
}
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
this.logger.info('CatalogService - Manage Data assets permissions');
if (!data.embed) data.embed = undefined;
@@ -235,34 +180,6 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -339,7 +256,7 @@ class CatalogService implements OnModuleInit {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
@@ -422,7 +339,6 @@ class CatalogService implements OnModuleInit {
} as typeof data_asset;
});
}
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
const { session } = await lastValueFrom(
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
@@ -435,176 +351,6 @@ class CatalogService implements OnModuleInit {
);
return res;
}
async addRlsRule(data: AddRlsRuleRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.AddRlsRule(data, metadata),
);
return res;
}
async removeRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogWriteService.RemoveRlsRule({ id }, metadata),
);
return res;
}
async batchRemoveRlsRule(
query: BatchRemoveRlsRulesRequest,
metadata: Metadata,
) {
const { id_rls, nimbus_dashboard_id } = query;
if (id_rls && nimbus_dashboard_id) {
throw new BadRequestException(
"You can't delete using both parameters. Choose either 'id_rls' or 'nimbus_dashboard_id'",
);
}
if (id_rls) {
await lastValueFrom(
this.catalogWriteService.RemoveRlsRulesByRlsId({ id_rls }, metadata),
);
} else if (nimbus_dashboard_id) {
await lastValueFrom(
this.catalogWriteService.RemoveRlsRulesByDashboardId(
{ nimbus_dashboard_id: parseInt(nimbus_dashboard_id) },
metadata,
),
);
}
return 'OK';
}
async getRlsRules(data: GetRlsRulesRequest, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetRlsRules(data, metadata),
);
return res.rls_rules;
}
async getOneRlsRule(id: number, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetOneRlsRule({ id }, metadata),
);
return res.rls_rule;
}
async getNimbusDashboards(
data: GetNimbusDashboardsRequest,
metadata: Metadata,
) {
const res = await lastValueFrom(
this.catalogReadService.GetNimbusDashboards(data, metadata),
);
return res.dashboards;
}
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{...body.logMetadata},
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
async createColumnMetadata(body: any): Promise<number[]> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{...body.logMetadata},
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
error.response?.status
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{...body.logMetadata},
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{...body.logMetadata},
);
throw new Error(error.response?.data?.message || error.message);
}
}
async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) {
const customer_name_raw = metadata.get('customer_name');
const customer_name = customer_name_raw?.[0]?.toString();
if (!customer_name) {
throw new BadRequestException('Customer name not found in metadata');
}
const res = await lastValueFrom(
this.platformWriteService.CatalogDataAssets(
{
data_assets: [
{
data_asset_id: table_metadata_id.toString(),
customer_name: customer_name,
data_asset_type: 'dataset',
},
],
},
metadata,
),
);
return res;
}
}
export { CatalogService };
export { CatalogService };
-16
View File
@@ -312,19 +312,3 @@ export class GetDatasetCatalogTaskRes {
@ApiProperty()
updated_by: string;
}
export class BatchRemoveRlsRulesRequest {
@ApiPropertyOptional()
nimbus_dashboard_id?: string;
@ApiPropertyOptional()
id_rls?: string;
}
export type AssetReporter = {
id: string;
display_name: string;
data_asset_type: string;
created_at: string;
tags: string;
}
-8
View File
@@ -1,8 +0,0 @@
export class PiiDto {
database_name: string;
table_schema: string;
table_name: string;
column_name: string;
data_type: string;
pii_rules: string;
}
@@ -1,89 +0,0 @@
import {
Controller,
Get,
Inject,
Param,
Req,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ShareService } from './share.service';
import { Request } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog/data-asset/share')
@UseFilters(new GrpcToHttpExceptionFilter())
export class ShareController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogShareService: ShareService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
}
@Get('/:id/preview')
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@Get('/:id/docs')
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
}
-30
View File
@@ -1,30 +0,0 @@
import { Module } from "@nestjs/common";
import { CatalogClientConfiguration } from "../catalog-client";
import { ClientsModule } from "@nestjs/microservices";
import { RolesModule } from "src/modules/roles/roles.module";
import { UsersModule } from "src/modules/users/users.module";
import { CustomersModule } from "src/modules/customers/customers.module";
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
UsersModule,
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
exports: [ShareModule],
})
export class ShareModule {}
-275
View File
@@ -1,275 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
ProtoServices,
ReadService,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
ForbiddenException,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { CatalogClientConfiguration } from '../catalog-client';
import { ClientGrpc } from '@nestjs/microservices';
import { UsersService } from 'src/modules/users/users.service';
import { RolesService } from 'src/modules/roles/roles.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id
}))
try {
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
type: isJWT(id) ? 'assigned' : shareMetadata.type,
customer: shareMetadata.customerName
}
if (user) {
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
} else {
await this.mixpanelService.trackShare(request, mixpanelTracker);
}
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { data_asset } = await this.getOneDataAsset({
customer_id: shareMetadata.customerId,
id: shareMetadata.assetId,
metadata,
});
this.logger.info('found asset: ' + JSON.stringify(data_asset));
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public') throw new NotFoundException();
return { data_asset };
} catch (error) {
this.logger.error(error);
throw error;
}
}
private async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: undefined },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
private async getAssetsUsersAndRoles(
data_assets: Array<any>,
customer_id: string,
) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
if (isJWT(id) && info ){
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return info;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -6,9 +6,7 @@ import {
} from '@nestjs/microservices';
import { ConnectionTest } from '@dadosfera/protospack-v2';
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
export class ConnectionTestClientConfiguration {
private config: GrpcOptions = {
@@ -20,7 +20,7 @@ import {
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@Injectable()
export class ConnectionTestService {
+1 -3
View File
@@ -6,9 +6,7 @@ import {
} from '@nestjs/microservices';
import { ConnectionManager } from '@dadosfera/protospack-v2';
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
export class ConnectionClientConfiguration {
public name = 'ConnectionClientConfiguration';
@@ -24,11 +24,10 @@ import {
ConnectionDetailsRes,
ConnectionRes,
ConnectionsRes,
GetAllConnectionsReq,
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -139,7 +138,7 @@ export class ConnectionController {
async getAllConnections(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Query() queries: GetAllConnectionsReq,
@Query() queries,
): Promise<ConnectionsRes> {
this.logger.info('/connections - Get All Connection');
+2 -14
View File
@@ -9,7 +9,6 @@ import {
ConnectionToCatalog,
} from '@dadosfera/protospack-v2/dist/lib/ConnectionManager/interfaces/entities';
import {
GetAllConnectionRequest,
GetAllConnectionResponse,
GetConnectionDetailsResponse,
GetConnectionResponse,
@@ -22,7 +21,7 @@ export type ConnectionCredentialsType =
| 'oauth'
| 'api_key'
| 'service_account'
| 'headers_auth';
| 'headers_authF';
export interface ConnectionApiConnection {
config_id: string;
plugin: string;
@@ -95,6 +94,7 @@ export class ConnectionDto implements Connection {
export class ConnectionToCatalogDto implements ConnectionToCatalog {
// ---Automatically generated information - will not be sent by the frontend--- //
id: string;
customer_id: string;
updated_at: string;
created_at: string;
@@ -104,9 +104,6 @@ export class ConnectionToCatalogDto implements ConnectionToCatalog {
customer_name: string;
// ---Information sent by the frontend--- //
@ApiPropertyOptional()
id: string;
@ApiProperty()
name: string;
@@ -150,15 +147,6 @@ export class UpdateConnectionDto extends PickType(ConnectionDto, [
'properties',
]) {}
export class GetAllConnectionsReq implements GetAllConnectionRequest {
@ApiPropertyOptional()
search?: string;
@ApiPropertyOptional()
page?: string;
@ApiPropertyOptional()
size?: string;
}
export class ConnectionsRes implements GetAllConnectionResponse {
@ApiProperty({ type: [ConnectionToCatalogDto] })
connections: ConnectionToCatalogDto[];
+1 -3
View File
@@ -6,9 +6,7 @@ import {
} from '@nestjs/microservices';
import { ConnectorManager } from '@dadosfera/protospack-v2';
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
export class ConnectorClientConfiguration {
public name = 'ConnectorClientConfiguration';
+13 -78
View File
@@ -1,35 +1,32 @@
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import {
Body,
Controller,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Query,
UseFilters,
} from '@nestjs/common';
import { ApiOkResponse, ApiProduces, ApiTags } from '@nestjs/swagger';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { ApiTags } from '@nestjs/swagger';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { CustomersService } from './customers.service';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { EnforceMfa } from './dtos/enforce-mfa';
@ApiInternalOnlyController()
@ApiTags('Customers')
@Controller('customers')
@Authenticated()
@UseFilters(GrpcToHttpExceptionFilter)
export class CustomersController {
logger: DadosferaLogger;
@@ -42,42 +39,26 @@ export class CustomersController {
this.logger = dadosferaLogger.logger;
}
@Post(':id/mfa')
@Authenticated()
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@RequireModule(DADOSFERA_MODULES_KEYS.DANGER_ZONE)
async enableMfaEnforce(@Param('id') id: string, @Body() data: EnforceMfa) {
this.logger.info('enableMfaEnforce', { id });
return await this.customersService.enableEnforceMfa(id, data.enabled);
}
@Get(':id/links')
@Authenticated()
@ApiOkResponse({ type: CustomerLinksResponse })
async getCustomerLinks(@Param('id') id: string) {
async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> {
this.logger.info('getCustomerLinks', { id });
const links = await this.customersService.getLinks(id);
return { links };
}
@Put(':id/links')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse()
@HttpCode(HttpStatus.OK)
async setCustomerLinks(
@Body() body: CustomerLinkRequest,
@Param('id') id: string,
) {
setCustomerLinks(
@Body() body: CustomerDto,
@Param('id') id,
): Promise<IdResponse> {
const { links } = body;
this.logger.info('setCustomerLinks', { id, links });
await this.customersService.setLinks(id, links);
return this.customersService.setLinks(id, links);
}
@Get('token')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN)
@ApiProduces('text/plain')
async getCustomerToken(
@Query('exp') exp: StringValue,
@User() user: RequestUser,
@@ -90,50 +71,4 @@ export class CustomersController {
};
return this.customersService.generateToken(exp, data);
}
@Get('monitoring-dashboard')
@Authenticated()
@RequireAllPermissions(
PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD,
)
async getCustomerMonitoringDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getCustomerMonitoringDashboard');
const metadata = PackTheMetadata(user);
return this.customersService.getMonitoringDashboardUrl(metadata);
}
@Get('logs-dashboard')
@Authenticated()
@RequireAllPermissions(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
)
@RequireModule(DADOSFERA_MODULES_KEYS.LOG_DASHBOARD)
async getCustomerMixPanelLogsDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getLogsDashboardUrl');
const metadata = PackTheMetadata(user);
const result = await this.customersService.getLogsDashboardUrl(metadata);
return result;
}
@Get('access-dashboard')
@Authenticated()
@RequireAllPermissions(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN,
)
@RequireModule(DADOSFERA_MODULES_KEYS.ACCESS_DASHBOARD)
async getAccessDashboard(
@User() user: RequestUser,
): Promise<{ url: string }> {
this.logger.info('getAccessDashboard');
const metadata = PackTheMetadata(user);
const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
return result;
}
}
+2 -9
View File
@@ -4,18 +4,11 @@ import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import { CustomersController } from './customers.controller';
import { CustomersService } from './customers.service';
import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
const ducClient = new DucClient();
const piFactoryClient = new PipelinesClientConfiguration();
const client = new DucClient();
@Module({
imports: [
ClientsModule.register([
ducClient.providerOptions,
piFactoryClient.providerOptions,
]),
],
imports: [ClientsModule.register([client.providerOptions])],
controllers: [CustomersController],
providers: [CustomersService, DadosferaLogger],
exports: [CustomersService],
+1 -104
View File
@@ -5,7 +5,6 @@ import {
HttpException,
HttpStatus,
InternalServerErrorException,
ForbiddenException,
} from '@nestjs/common';
import { firstValueFrom, lastValueFrom } from 'rxjs';
@@ -23,48 +22,20 @@ import {
} from '@aws-sdk/client-secrets-manager';
import getEnv from 'src/utils/getEnv';
import { logger } from 'elastic-apm-node';
import {
ReadService,
ProtoServices as PipelineProtoServices,
} from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { Metadata } from '@grpc/grpc-js';
import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
// This function will accept any string, which may result in a bug.
@Injectable()
export class CustomersService implements OnModuleInit {
private customerService: CustomersProtoService;
private logger: DadosferaLogger;
private pipelineReadService: ReadService.PipelineV2ReadService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
@Inject(PipelinesClientConfiguration.name)
private readonly pipelinesGrpcClient: ClientGrpc,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
) {}
onModuleInit() {
this.customerService = this.grpcClient.getService<CustomersProtoService>(
ProtoServices.CustomersProtoService,
);
this.pipelineReadService =
this.pipelinesGrpcClient.getService<ReadService.PipelineV2ReadService>(
PipelineProtoServices.PipelineV2ReadService,
);
}
async getCustomer(customerId: string) {
return await lastValueFrom(
this.customerService.CustomerFindOneById({
id: customerId
})
)
}
async getLinks(customerId: string) {
@@ -149,78 +120,4 @@ export class CustomersService implements OnModuleInit {
// const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token;
}
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
const res = await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: '98',
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata,
),
);
logger.info('Done');
return res;
}
async getLogsDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
const res = await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: '103',
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata,
),
);
logger.info('Done');
return res;
}
async getAccessDashboardUrl(customerName: string, metadata: Metadata) {
/*
* TODO(Refactor): dar um jeito de exibir o dash da sbm diferente dos outros customer
* pois o signicado de department para sbm significa as instituições do usuários
*/
if (customerName !== 'sbmoffshorecom') {
throw new ForbiddenException();
}
logger.info('CustomersService - getAccessDashboardUrl');
const res = await this.getDashboardUrl('105', metadata);
logger.info('Done');
return res;
}
private async getDashboardUrl(dashboardId: string, metadata: Metadata) {
return await lastValueFrom(
this.pipelineReadService.PipelineV2GetDashboardUrl(
{
dashboard_id: dashboardId,
exp: '15m',
metabase_customer_name: 'dadosferatech',
},
metadata
)
);
}
async enableEnforceMfa(id: string, enabled: boolean) {
return await lastValueFrom(
this.customerService.CustomerUpdateEnforceMfa({
customerId: id,
enforceMfa: enabled
})
)
}
}
+5 -16
View File
@@ -1,23 +1,12 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { ApiProperty } from '@nestjs/swagger';
export class CustomerLink implements Link {
export class CustomerDto {
@ApiProperty()
href: string;
@ApiProperty()
name: string;
@ApiProperty()
description: string;
@ApiPropertyOptional()
iconSrc: string;
}
export class CustomerLinkRequest {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
links: Link[];
}
export class CustomerLinksResponse {
@ApiProperty({ type: [CustomerLink] })
links: CustomerLink[];
@ApiProperty()
links: Link[];
}
@@ -1,6 +0,0 @@
import { ApiProperty } from "@nestjs/swagger";
export class EnforceMfa {
@ApiProperty()
enabled: boolean
}
+1 -5
View File
@@ -9,9 +9,7 @@ import {
ProtoPaths,
} from '@dadosfera/protospack-v2/dist/lib/Duc';
const isLocalConnection =
process.env.DUC_URL.startsWith('duc:') ||
process.env.DUC_URL.includes('0.0.0.0');
const isLocalConnection = !!process.env.DUC_URL?.includes('0.0.0.0');
export class DucClient {
public name = 'DucClient';
@@ -29,8 +27,6 @@ export class DucClient {
objects: true,
arrays: true,
},
maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem
maxReceiveMessageLength: 15 * 1024 * 1024,
},
};
@@ -1,47 +0,0 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateIdentityProvider {
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
clientSecret: string;
@ApiProperty()
issuerUrl: string;
@ApiProperty()
permissions: number[];
}
export class IdentityProviderResponse {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
issueUrl: string;
@ApiProperty()
permissions: {
id: number;
name: string;
}[];
}
export class IdentityProviderListResponse {
@ApiProperty()
providers: IdentityProviderResponse[]
}
@@ -1,10 +0,0 @@
export class SsoSignInDto {
readonly nonce: string;
readonly codeVerifier: string;
readonly state: string;
readonly id: string;
readonly clientId: string;
readonly clientSecret: string;
readonly issuerUrl: string;
readonly redirectUrls: string[];
}
@@ -1,246 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Redirect,
Req,
} from '@nestjs/common';
import { IdentityProviderService } from './identity-provider.service';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiOkResponse } from '@nestjs/swagger';
import {
CreateIdentityProvider,
IdentityProviderListResponse,
IdentityProviderResponse,
} from './dto/identity-provider.dto';
import { Request } from 'express';
import ErrorCodes from 'src/utils/errorCodes';
import {
Authenticated,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('identity-providers')
export class IdentityProviderController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private identityProviderService: IdentityProviderService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async addIdentityProvider(
@User() user: RequestUser,
@Body() body: CreateIdentityProvider,
@Language() language: LanguageEnum,
) {
this.logger.info('POST /identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
return await this.identityProviderService.create(body, metadata);
}
@Get()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderListResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async getProviders(
@User() user: RequestUser,
@Language() language: LanguageEnum,
) {
this.logger.info('GET identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
const result = await this.identityProviderService.getList(metadata);
return result;
}
@Delete(':id')
@HttpCode(HttpStatus.NO_CONTENT)
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async deleteIdentityProvider(
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('DELETE /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.deleteIdentityProvider(
id,
metadata,
);
}
@Put(':id')
@HttpCode(HttpStatus.OK)
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async updateIdentityProviders(
@Param('id') id: string,
@Body() body: CreateIdentityProvider,
@User() user: RequestUser,
) {
this.logger.info('PUT /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.updateIdentityProviders(
id,
body,
metadata,
);
}
@Post('/callback')
@HttpCode(HttpStatus.OK)
async callbackIdp(
@Req() req: Request,
@Language() language: LanguageEnum,
@Body()
body: {
state: string;
code: string;
},
) {
this.logger.info('GET /identity-providers/callback');
const { code, state } = body;
if (!code) {
this.logger.error('No code received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
if (!state) {
this.logger.error('No state received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@HttpCode(HttpStatus.OK)
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -1,16 +0,0 @@
import { Module } from '@nestjs/common';
import { IdentityProviderController } from './identity-provider.controller';
import { IdentityProviderService } from './identity-provider.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ServicesModule } from 'src/services/service.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
controllers: [IdentityProviderController],
providers: [IdentityProviderService, DadosferaLogger],
})
export class IdentityProviderModule {}
@@ -1,185 +0,0 @@
import {
BadRequestException,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
client_secret: idp.clientSecret,
redirect_uris: idp.redirectUrls,
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
id: idp.id,
state,
clientId: idp.clientId,
clientSecret: idp.clientSecret,
issuerUrl: idp.issuerUrl,
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
code_challenge,
code_challenge_method: 'S256',
state,
nonce,
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
client_secret: ssoSign.clientSecret,
redirect_uris: ssoSign.redirectUrls,
});
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
idToken: tokenSet.id_token,
refreshToken: tokenSet.refresh_token,
id: ssoSign.id,
}),
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
}
async updateIdentityProviders(
id: string,
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
id,
...body,
},
metadata,
),
);
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
import { Info } from 'protospack/dist/lib/interfaces';
interface Values {
jdbc_user: string;
+1 -3
View File
@@ -6,9 +6,7 @@ import {
type GrpcOptions,
} from '@nestjs/microservices';
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
export class InputsGrpcClient {
public readonly name = 'InputsGrpcClient';
private config: GrpcOptions = {
+77 -11
View File
@@ -1,29 +1,95 @@
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
constructor(
private mixpanelService: MixpanelService
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
) {}
@Post(':id')
async trackEvent(
@Param('id') id,
@Body() body,
@User() user: RequestUser,
@Req() request
) {
async trackEvent(@Param('id') id, @Body() body, @User() user: RequestUser) {
delete body.info;
await this.mixpanelService.track(id, user, request, body)
//TODO: remove this when mixpanel track is working
return true;
const mixpanel = init(this.mixpanelToken);
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
...body,
});
return { id, body, user: user.username };
}
capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
+1 -6
View File
@@ -1,8 +1,6 @@
import { Module } from '@nestjs/common';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
import { MixpanelService } from './mixpanel.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [MixpanelController],
@@ -10,12 +8,9 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs';
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecretFromSecretsManager(
`prd/root/mixpanel_token`,
`${process.env.ENV}/root/mixpanel_token`,
),
},
MixpanelService,
DadosferaLogger
],
exports: [MixpanelService]
})
export class MixpanelModule {}
-118
View File
@@ -1,118 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Inject } from '@nestjs/common';
import { Request } from 'express';
import mixpanel, { init } from 'mixpanel';
import { RequestUser } from 'src/decorators/user.decorator';
export class MixpanelService {
logger: DadosferaLogger;
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async track(eventName: string, user: RequestUser, request: Request, body: any) {
this.logger.info("track: " + JSON.stringify({
eventName,
...body
}))
const mixpanel = init(this.mixpanelToken);
await this.setPeople(user, mixpanel);
await mixpanel.track(eventName, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
async trackShare(request: Request, body: any) {
this.logger.info("trackShare: " + JSON.stringify(body))
const mixpanel = init(this.mixpanelToken);
await mixpanel.track("share_page", {
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
}
private capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
private getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
@@ -6,9 +6,7 @@ import {
} from '@nestjs/microservices';
import { NetworkConfig } from '@dadosfera/protospack-v2';
const isLocalConnection =
process.env.INFACTORY_URL.startsWith('in-factory:') ||
process.env.INFACTORY_URL.includes('0.0.0.0');
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
export class NetworkConfigGrpcClient {
public name = 'NetworkConfigGrpcClient';
@@ -1,6 +0,0 @@
import { ApiProperty } from "@nestjs/swagger";
export class NetworkPoliciesDTO {
@ApiProperty()
policies: string []
}
@@ -1,64 +0,0 @@
import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Post } from '@nestjs/common';
import { ApiOkResponse } from '@nestjs/swagger';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/decorators/authentication.decorator';
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { NetworkPolicyService } from './network-policy.service';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('network-policy')
export class NetworkPolicyController {
constructor(private networkPolicyService: NetworkPolicyService) {}
@Get()
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
async getNetworks(
@User() user: RequestUser,
) {
return await this.networkPolicyService.getByCustomer(
user.customer_id
);
}
@Post()
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse()
@HttpCode(HttpStatus.CREATED)
async applyNetworkPolicies(
@User() user: RequestUser,
@Body() data: NetworkPoliciesDTO,
) {
const metadata = PackTheMetadata(user);
return await this.networkPolicyService.apply(
data.policies,
user.customer_id,
metadata,
);
}
@Delete()
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse()
@HttpCode(HttpStatus.OK)
async removeNetworkPolicies(
@User() user: RequestUser,
@Body() data: NetworkPoliciesDTO,
) {
const metadata = PackTheMetadata(user);
return await this.networkPolicyService.delete(
data.policies,
user.customer_id,
metadata,
);
}
}
@@ -1,19 +0,0 @@
import { Module } from '@nestjs/common';
import { NetworkPolicyController } from './network-policy.controller';
import { NetworkPolicyService } from './network-policy.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const ducClient = new DucClient();
@Module({
imports: [
ClientsModule.register([
ducClient.providerOptions
]),
],
controllers: [NetworkPolicyController],
providers: [NetworkPolicyService, DadosferaLogger]
})
export class NetworkPolicyModule {}
@@ -1,74 +0,0 @@
import { Inject, Injectable } from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
@Injectable()
export class NetworkPolicyService {
private customerService: CustomersProtoService;
private logger: DadosferaLogger;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.customerService = this.grpcClient.getService<CustomersProtoService>(
ProtoServices.CustomersProtoService,
);
}
async getByCustomer(id: string): Promise<NetworkPoliciesDTO> {
const {
customer
} = await lastValueFrom(this.customerService.CustomerFindOneById({
id
}));
return {
policies: customer.networkPolicies
}
}
async apply(
networkPolicies: string[],
customerId: string,
metadata: Metadata,
) {
return await lastValueFrom(
this.customerService.CustomerCreateNetworkPolicy(
{
customerId,
networkPolicies,
},
metadata,
),
);
}
async delete(
networkPolicies: string[],
customerId: string,
metadata: Metadata,
) {
return await lastValueFrom(
this.customerService.CustomerRemoveNetworkPolicy(
{
customerId,
networkPolicies,
},
metadata,
),
);
}
}
+1 -1
View File
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
@@ -1,48 +0,0 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateUserOpenDataDTO {
@ApiProperty()
firstName: string;
@ApiProperty()
lastName: string;
@ApiProperty()
email: string;
@ApiProperty()
organization: string;
@ApiProperty()
enquiryType: string;
}
type FormField = {
id: string;
type: string;
title: string;
value: string;
raw_value: string;
required: string;
};
type MetaData = {
title: string;
value: string;
};
export type WordpressForm = {
form: {
id: string;
name: string;
};
fields: {
[key: string]: FormField;
};
meta: {
date: MetaData;
time: MetaData;
page_url: MetaData;
user_agent: MetaData;
remote_ip: MetaData;
credit: MetaData;
};
};
@@ -1,101 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Body, Controller, ForbiddenException, Header, Headers, HttpCode, HttpException, Inject, Param, Post, Query, Req, Res, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common';
import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { LanguageEnum } from 'src/utils/languages.enum';
import { UsersService } from '../users/users.service';
import { Language } from 'src/decorators/language.decorator';
import { OpenDataService } from './open-data.service';
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { request } from 'http';
import { Request } from 'express';
@Controller('open-data')
@ApiInternalOnlyController()
@ApiTags('OpenData')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(GrpcToHttpExceptionFilter)
export class OpenDataController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private openDataService: OpenDataService,
) {
this.logger = dadosferaLogger.logger;
}
@Post("/sharing-ocean-data")
@HttpCode(200)
@Header('content-type', 'application/json')
@ApiOkResponse()
async createUser(
@Body()
body: WordpressForm,
@Query('language')
language: string,
@Req()
request: Request,
@Headers('origin')
origin: string
) {
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
// if (!corslist.includes(origin)) {
// this.logger.info('block request by cors list: '+ origin);
// throw new ForbiddenException();
// }
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
const roles = [process.env.OPEN_GROUP_ID];
const metadata = PackTheMetadata({
language: language || 'en-us'
});
const data = {}
try {
Object.keys(body.fields)
.filter(key => body.fields[key].required === "1")
.forEach(key => {
const field = body.fields[key]
data[field.id] = field.value
});
} catch (e) {
this.logger.error('user data ' + e.message);
}
const user: CreateUserOpenDataDTO = {
email: data["email"],
enquiryType: data["enquiry_type"],
firstName: data["first_name"],
lastName: data["last_name"],
organization: data["organization"]
}
this.logger.info(`user request to group ${OPENDATA_CUSTOMER_ID} with role ${OPENDATA_GROUP_ID}`);
try {
const id = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata);
this.logger.info('user created with id: '+ id);
return {
success: true,
status: 'success',
message: 'user created with succesfull'
}
} catch (e) {
this.logger.error('failed with exception: ' + e.message);
return {
success: false,
status: 'failed',
message: e.message
};
}
}
}
-22
View File
@@ -1,22 +0,0 @@
import { Module } from '@nestjs/common';
import { OpenDataController } from './open-data.controller';
import { UsersService } from '../users/users.service';
import { ClientsModule } from '@nestjs/microservices'
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
import { OpenDataService } from './open-data.service';
const client = new DucClient();
@Module({
controllers: [OpenDataController],
imports: [
ClientsModule.register([client.providerOptions]),
RolesModule,
// PermissionsModule,
],
providers: [DadosferaLogger, UsersService, OpenDataService]
})
export class OpenDataModule {}
@@ -1,51 +0,0 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { lastValueFrom } from 'rxjs';
import { CreateUserOpenDataDTO } from './dto/wordpres-form';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { Metadata } from '@grpc/grpc-js';
@Injectable()
export class OpenDataService implements OnModuleInit {
logger: DadosferaLogger;
private usersClientService: UsersProtoService;
constructor(
@Inject(DadosferaLogger)
private dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name)
private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.usersClientService = this.grpcClient.getService(
ProtoServices.UsersProtoService,
);
}
async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) {
const body = {
email: data.email,
name: data.firstName + " " + data.lastName,
department: data.organization,
jobTitle: data.enquiryType,
customerId: customerId,
roleIds: roleIds
}
try {
const { user } = await lastValueFrom(
this.usersClientService.SimpleUserCreate(body, metadata),
);
return user.id;
} catch(err) {
return err;
}
}
}
+2 -6
View File
@@ -1,9 +1,6 @@
import { ConflictException, Inject, OnModuleInit } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import {
PipelineServicesNames,
PipelinesServiceInterface,
} from '@dadosfera/protospack';
import { PipelineServicesNames, PipelinesServiceInterface } from 'protospack';
import { lastValueFrom } from 'rxjs';
import { IIdRequest } from './interfaces';
@@ -14,10 +11,9 @@ import { PipelinesClientConfiguration } from './pipelines-client';
export class PipelinesClientService implements OnModuleInit {
private pipelineService: PipelinesServiceInterface;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
dadosferaLogger: DadosferaLogger,
@Inject(PipelinesClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
) {
+1 -1
View File
@@ -1,4 +1,4 @@
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
import { Info } from 'protospack/dist/lib/interfaces';
export interface ICreatePipelineDto {
input: IdRequest;

Some files were not shown because too many files have changed in this diff Show More