Compare commits

..
Author SHA1 Message Date
marcos-silva-rodrigues f3550255c4 FIX: validate if permission is public 2025-06-11 15:18:55 -03:00
112 changed files with 3035 additions and 11612 deletions
-12
View File
@@ -1,12 +0,0 @@
node_modules
dist
.git
*.log
npm-debug.log*
.DS_Store
.env
.env.*
coverage
.nyc_output
*.tgz
!protospack.tgz
+72 -7
View File
@@ -13,6 +13,11 @@ on:
options:
- stg
- prd
push_to_dockerhub:
description: "Push image to Dockerhub?"
required: true
type: boolean
default: false
jobs:
extract_environment:
@@ -113,6 +118,23 @@ jobs:
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
- name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
uses: docker/login-action@v2
with:
username: dadosfera
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build, Tag, and Push Image to Dockerhub
if: ${{inputs.push_to_dockerhub}}
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker compose -f build.docker-compose.dockerhub.yml build
docker compose -f build.docker-compose.dockerhub.yml push
# - name: Create ZIP file to Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
@@ -142,11 +164,54 @@ jobs:
docker system prune --volumes -a -f
docker system df
k8s-deploy:
helmfile-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: 'oracle'
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Apply
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
-137
View File
@@ -1,137 +0,0 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+25 -51
View File
@@ -4,7 +4,7 @@ on:
pull_request:
branches:
- main
- beta
- stg
jobs:
extract_environment:
@@ -21,13 +21,17 @@ jobs:
fi
id: extract_environment
helmfile-check:
env:
HOME: /home/runner
helmfile-deploy:
needs: [extract_environment]
environment: ${{ needs.extract_environment.outputs.environment }}
runs-on: [self-hosted, "prd-oracle"]
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Summary
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
- name: Checkout code
uses: actions/checkout@v3
@@ -36,28 +40,13 @@ jobs:
with:
version: 'v3.9.0'
- name: Determine DNS_HOST based on environment
id: set_dns
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
- name: Install Azure ClI
run: |
if [ "$ENV" = "prd" ]; then
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
elif [ "$ENV" = "stg" ]; then
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
fi
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Set up Python
uses: actions/setup-python@v4
@@ -66,40 +55,25 @@ jobs:
- name: Install Helmfile
run: |
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
mv helmfile /usr/local/bin/
helmfile --version
- name: Debug Helm env
run: |
helm env
echo "HOME=$HOME"
ls -R $HOME/.local/share/helm || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
run: helmfile -f helmfiles/${ENV}.yaml diff
+1 -2
View File
@@ -1,5 +1,4 @@
FROM node:20-alpine AS base_image
RUN npm install -g npm@latest
FROM node:18.17-alpine AS base_image
FROM base_image AS build_base
WORKDIR /app
-47
View File
@@ -1,47 +0,0 @@
FROM node:22-alpine AS base_image
RUN npm install -g npm@latest
FROM base_image AS build_base
WORKDIR /app
RUN apk update
RUN apk add --no-cache \
aws-cli \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
COPY package*.json ./
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# Local build with secrets
FROM build_base AS build
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
RUN npm ci
COPY . .
RUN npm run build
FROM base_image
WORKDIR /app
COPY --from=build /app/dist ./dist
COPY --from=build /app/node_modules ./node_modules
COPY --from=build /app/package*.json ./
RUN apk update
RUN apk add --no-cache \
chromium \
nss \
freetype \
harfbuzz \
ca-certificates \
ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
ENTRYPOINT ["npm", "run", "start:prod"]
+1
View File
@@ -2,6 +2,7 @@
<image src="./assets/maestro.svg" style="width:10rem">
</p>
# Maestro
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
@@ -1,41 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
pathType: Prefix
-36
View File
@@ -1,36 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
-18
View File
@@ -1,18 +0,0 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity: null
-56
View File
@@ -1,56 +0,0 @@
releases:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro-unimed.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
# 10.70.0.0/16 internal network
# 137.131.167.254/32 loadbalancer
# 159.112.184.81/32 cluster ip for the uptime request ingest
- name: maestro.restricted_ip
value: "177.52.172.0/24, 189.84.160.157/32, 186.237.171.146/32, 137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
-30
View File
@@ -1,30 +0,0 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
- name: private-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
set:
- name: app_name
value: maestro-private
- name: hostname
value: private-maestro.stg.dadosfera.ai
# Customer id
- name: maestro.dedicated_proxy
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
- name: replicaCount
value: 1
- name: affinity
value: null
- name: resources
value: null
- name: maestro.restricted_ip
value: "137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+673 -3115
View File
File diff suppressed because it is too large Load Diff
-3
View File
@@ -14,9 +14,6 @@ declare global {
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
REDIS_TLS?: string;
}
}
}
+22
View File
@@ -0,0 +1,22 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
+22
View File
@@ -0,0 +1,22 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.stg.dadosfera.ai
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
- name: replicaCount
value: 1
+23
View File
@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
@@ -1,16 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.app_name }}
name: maestro
namespace: applications
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: {{ .Values.app_name }}
app: maestro
strategy:
rollingUpdate:
@@ -20,17 +20,22 @@ spec:
template:
metadata:
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
imagePullSecrets:
- name: {{ .Values.imagePullSecrets }}
nodeSelector:
"beta.kubernetes.io/os": linux
{{- if .Values.affinity }}
affinity:
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: application
operator: In
values:
- general
tolerations:
- key: "kubernetes.azure.com/scalesetpriority"
@@ -43,14 +48,14 @@ spec:
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
ports:
- containerPort: {{ .Values.containerPort }}
{{- if .Values.resources }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
requests:
cpu: {{ .Values.resources.requests.cpu }}
memory: {{ .Values.resources.requests.memory }}
limits:
cpu: {{ .Values.resources.limits.cpu }}
memory: {{ .Values.resources.limits.memory }}
env:
# Auth Provider Configuration (cognito or keycloak)
- name: AUTH_PROVIDER
value: {{ .Values.maestro.auth_provider | default "cognito" | quote }}
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
- name: AWS_REGION
@@ -77,8 +82,6 @@ spec:
value: "logstash-pipelines.dadosfera.ai"
- name: LOGGER_GELF_PORT
value: "{{ .Values.maestro.logger_gelf_port }}"
- name: LOGGER_CONSOLE_EXTRA
value: "true"
- name: NIMBUS_BASE_URL
value: "http://nimbus-api"
- name: NPM_TOKEN
@@ -97,22 +100,6 @@ spec:
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: REDIS_TLS
value: "{{ .Values.maestro.redis_tls }}"
- name: PLATFORM_API_URL
value: {{ .Values.maestro.platform_api_url }}
- name: STORAGE_EXPLORER_API_URL
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
@@ -121,26 +108,15 @@ spec:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
name: prd-maestro
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
name: prd-maestro
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
name: prd-maestro
key: AWS_DEFAULT_REGION
# Elasticsearch
- name: ELASTICSEARCH_URL
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: ELASTICSEARCH_URL
- name: ELASTICSEARCH_API_KEY
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
key: ELASTICSEARCH_API_KEY
+28
View File
@@ -0,0 +1,28 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: maestro
name: open-data
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: maestro
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
pathType: Prefix
+27
View File
@@ -0,0 +1,27 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: maestro
name: maestro
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: maestro
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
@@ -1,17 +1,17 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-{{ .Values.app_name }}
name: prd-maestro
namespace: applications
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-{{ .Values.app_name }}
name: prd-maestro
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
@@ -38,15 +38,3 @@ spec:
version: "AWSCURRENT"
property: token
- secretKey: ELASTICSEARCH_URL
remoteRef:
key: {{ .Values.maestro.env }}/microservices/elasticsearch
version: "AWSCURRENT"
property: ELASTICSEARCH_URL
- secretKey: ELASTICSEARCH_API_KEY
remoteRef:
key: {{ .Values.maestro.env }}/microservices/elasticsearch
version: "AWSCURRENT"
property: ELASTICSEARCH_API_KEY
@@ -1,18 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.app_name }}
name: maestro
namespace: applications
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
type: ClusterIP
ports:
- name: {{ .Values.app_name }}
- name: maestro
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: {{ .Values.app_name }}
app: maestro
@@ -9,7 +9,6 @@ image:
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
@@ -27,9 +26,6 @@ resources:
cpu: 2000m
memory: 2Gi
maestro:
# Auth provider: "cognito" (default) or "keycloak"
# Note: maestro doesn't connect to Keycloak directly, only duc does
auth_provider: "cognito"
aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
duc_url: duc.dadosfera.ai
in_factory_url: in-factory.dadosfera.ai
@@ -46,28 +42,9 @@ maestro:
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
dedicated_proxy: ""
restricted_ip: ""
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
redis_port: "6379"
redis_database: "0"
redis_tls: "true"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- product
+1734 -2806
View File
File diff suppressed because it is too large Load Diff
+3 -22
View File
@@ -27,15 +27,10 @@
"test:e2e": "jest --config ./test/jest-e2e.json"
},
"dependencies": {
"@aws-crypto/sha256-js": "^5.2.0",
"@aws-sdk/client-dynamodb": "^3.414.0",
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@aws-sdk/credential-provider-node": "^3.940.0",
"@aws-sdk/lib-dynamodb": "^3.414.0",
"@aws-sdk/signature-v4": "^3.370.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "^3.38.0-beta.26",
"@dadosfera/protospack-v2": "3.38.0-beta.1",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -49,12 +44,9 @@
"@nestjs/schematics": "^9.2.0",
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.30.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"axios": "^0.27.2",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -65,7 +57,6 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
@@ -73,26 +64,16 @@
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.6.3"
},
"overrides": {
"multer": "2.0.2",
"form-data": "^4.0.4",
"body-parser": "^1.20.3",
"cross-spawn": "^7.0.5",
"glob": "^10.5.0",
"path-to-regexp": "^3.3.0",
"semver": "^7.5.2"
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/cache-manager": "^4.0.6",
"@types/cookie-parser": "^1.4.9",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
+2 -14
View File
@@ -28,13 +28,8 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
@Module({
providers: [
@@ -68,17 +63,10 @@ import { StorageExplorerModule } from './modules/storage-explorer/storage-explor
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
NetworkPolicyModule,
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
PlatformApiModule,
StorageExplorerModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
NetworkPolicyModule,
ApiKeyModule
],
})
export class AppModule {}
+12 -20
View File
@@ -17,7 +17,6 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -100,7 +99,6 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -122,12 +120,6 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -448,18 +440,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
@@ -4,7 +4,6 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -136,18 +135,6 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
-20
View File
@@ -1,20 +0,0 @@
import jwt, { JwtPayload } from 'jsonwebtoken';
export function extractUserFrom(aRawJwt: string) {
const decodedToken = jwt.decode(aRawJwt, {
complete: true,
});
const payload = decodedToken.payload as JwtPayload;
return {
user_id: payload.user_id,
username: payload.username,
permissions: payload.permissions,
customer_id: payload.customer_id,
customer_name: payload.customer_name,
customer_tier: payload.customer_tier,
customer_modules: payload.customer_modules,
access_token: aRawJwt,
}
}
+27 -108
View File
@@ -116,44 +116,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
IMPORT_FILES: {
title: {
'pt-br': 'Coletar | Importar arquivos',
'en-us': 'Collect | Import files',
'es-es': 'Colecta | Importar archivos',
},
permissions: {
VIEW: {
seqid: 48,
claim: 'import-file:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Importar arquivos',
'en-us': 'Import files',
'es-es': 'Importar archivos',
},
},
},
},
AI_CHAT: {
title: {
'pt-br': 'AutodriveDDF',
'en-us': 'AutodriveDDF',
'es-es': 'AutodriveDDF',
},
permissions: {
VIEW: {
seqid: 49,
claim: 'ai-chat:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'AutodriveDDF',
'en-us': 'AutodriveDDF',
'es-es': 'AutodriveDDF',
},
},
},
},
CONNECTION: {
title: {
'pt-br': 'Coletar | Fontes de dados',
@@ -378,6 +340,16 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
@@ -390,44 +362,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
LINEAGE: {
title: {
'pt-br': 'Explorar | Linhagem',
'en-us': 'Explore | Lineage',
'es-es': 'Explorar | Linaje',
},
permissions: {
VIEW: {
seqid: 50,
claim: 'lineage:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar ao módulo de Linhagem',
'en-us': 'Access to Lineage module',
'es-es': 'Acceda al módulo de Linaje',
},
}
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -668,36 +602,23 @@ export const PERMISSIONS_GROUPS = {
},
},
},
STORAGE_EXPLORER: {
title: {
'pt-br': 'Storage Explorer',
'en-us': 'Storage Explorer',
'es-es': 'Storage Explorer',
},
permissions: {
READ: {
seqid: 51,
claim: 'storage-explorer:read',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Ler dados do Storage Explorer',
'en-us': 'Read Storage Explorer data',
'es-es': 'Leer datos del Storage Explorer',
},
},
WRITE: {
seqid: 52,
claim: 'storage-explorer:write',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Escrever dados no Storage Explorer',
'en-us': 'Write Storage Explorer data',
'es-es': 'Escribir datos en Storage Explorer',
},
},
},
},
};
export const PUBLIC_PERMISSIONS_SEQID = [
46, // AUTH.GENERATE_TOKEN
23, 13, 29, 5, // PIPELINE
37, 38, // CONNECTION
35, // NETWORK_CONFIG
7, 19, 25, 1, // CATALOG
14, // SNOWFLAKE
11, // DATAVIZ
31, // INTELLIGENCE
34, // USERS
43, // PROCESS
47 // CUSTOMER
];
export interface DadosferaModule {
name: string;
description: string;
@@ -709,9 +630,7 @@ export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
PII: 'pii'
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+1 -9
View File
@@ -9,7 +9,6 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -53,7 +52,6 @@ describe('user.decorator', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -76,12 +74,6 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -183,5 +175,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
// UserTest(token);
UserTest(token);
});
+3 -27
View File
@@ -9,8 +9,6 @@ import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -18,41 +16,19 @@ async function bootstrap() {
});
const logger = new DadosferaLogger();
const corsOrigins = [];
if (process.env.ENV === 'local') {
corsOrigins.push('http://localhost:4200');
} else {
corsOrigins.push(
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://private-frontend.stg.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
);
}
const app = await NestFactory.create(AppModule, {
logger,
cors: {
origin: corsOrigins,
origin: '*',
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true,
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV !== 'local') {
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use(
'/catalog/register-dataset',
urlencoded({ extended: true, limit: '10mb' }),
);
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
}
configureSwagger(app);
+5 -1
View File
@@ -1,12 +1,13 @@
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { Authenticated, RequireAllPermissions } from 'src/decorators/authentication.decorator';
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('api-key')
@Authenticated()
@@ -26,6 +27,7 @@ export class ApiKeyController {
@Post()
@ApiResponse({ type: CreateApiKeyResponseDto })
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
this.logger.info('POST /api-key', {
permissions: createApiKeyDto.permissions,
@@ -41,6 +43,7 @@ export class ApiKeyController {
@Get()
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
this.logger.info('GET /api-key', {
method: 'findAll'
@@ -54,6 +57,7 @@ export class ApiKeyController {
}
@Delete(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
this.logger.info('DELETE /api-key/:id', {
id,
+46 -11
View File
@@ -1,38 +1,73 @@
import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
import {
Injectable,
Inject,
OnModuleInit,
BadRequestException,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import {
CreateApiKeyDto,
CreateApiKeyResponseDto,
ApiKeyBaseResponseDto,
} from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { PackTheMetadata } from '../../utils/ PackTheMetadata';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PUBLIC_PERMISSIONS_SEQID } from 'src/authentication/permissions.enum';
@Injectable()
export class ApiKeyService implements OnModuleInit {
private apiKeyService: ApiKeyWriteProtoService;
constructor(
@Inject(DucClient.name) private readonly client: ClientGrpc,
@Inject(DucClient.name) private readonly client: ClientGrpc
) {}
onModuleInit() {
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(
ProtoServices.ApiKeyWriteProtoService,
);
}
create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
create(
createApiKeyDto: CreateApiKeyDto,
user: RequestUser,
): Promise<CreateApiKeyResponseDto> {
const metadata = PackTheMetadata(user);
return lastValueFrom(this.apiKeyService.CreateApiKey({
permissions: createApiKeyDto.permissions
}, metadata));
const invalidPermissions = [];
for (const permission of createApiKeyDto.permissions) {
if (!PUBLIC_PERMISSIONS_SEQID.includes(permission)) {
invalidPermissions.push(permission);
}
}
if (invalidPermissions.length > 0) {
throw new BadRequestException(
`Invalid permissions: ${invalidPermissions.join(', ')}`,
);
}
return lastValueFrom(
this.apiKeyService.CreateApiKey(
{
permissions: createApiKeyDto.permissions,
},
metadata,
),
);
}
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
const metadata = PackTheMetadata(user);
console.log(metadata)
console.log(metadata);
const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
const data = await lastValueFrom(
this.apiKeyService.ListApiKeys({}, metadata),
);
return data.api_keys;
}
-37
View File
@@ -1,37 +0,0 @@
import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('assign')
@Authenticated()
export class AssignController {
constructor(private readonly assignService: AssignService) {}
@Put('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
}
@Get('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
try {
return await this.assignService.get(metadata);
} catch (error) {
throw new NotFoundException(error.message)
}
}
}
-15
View File
@@ -1,15 +0,0 @@
import { Module } from '@nestjs/common';
import { AssignService } from './assign.service';
import { AssignController } from './assign.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AssignController],
providers: [AssignService, DadosferaLogger]
})
export class AssignModule {}
-38
View File
@@ -1,38 +0,0 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Metadata } from '@grpc/grpc-js';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from 'src/modules/duc/client.config';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
@Injectable()
export class AssignService implements OnModuleInit {
ducService: AssingProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<AssingProtoService>(
ProtoServices.AssingProtoService,
);
}
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
return data;
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -1,3 +0,0 @@
export class CreateAssignDto {
publicKey: string;
}
+16 -125
View File
@@ -12,8 +12,6 @@ import {
Redirect,
Req,
Param,
Res,
UnauthorizedException,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -28,7 +26,7 @@ import {
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest
AuthVerifyTotpMfaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -47,21 +45,14 @@ import {
AuthSignInRes,
BulkEditRequest,
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request, Response } from 'express';
import { Request } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt, { JwtPayload } from 'jsonwebtoken';
import jwt from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -75,7 +66,6 @@ export class AuthController {
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private authClient: AuthClientService,
private apiKeyService: ApiKeyService,
) {
this.logger = dadosferaLogger.logger;
@@ -96,46 +86,13 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.authClient.writeAuthSession(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
throw error;
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
this.authClient.cleanUpAuthSession(res);
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
@Headers('origin') origin = '',
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ language, custom_host: frontHost });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
return this.authClient.signIn({ username, password, totp }, metadata);
}
@Post('refresh-access-token')
@@ -145,26 +102,17 @@ export class AuthController {
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, customerName: customer_name } = body;
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const metadata = PackTheMetadata({
language,
customer_name,
custom_host: frontHost,
});
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.authClient.writeAuthSession(res, {
accessToken: data.accessToken,
refreshToken: data.refreshToken,
userId
});
return res.send(data);
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
}
@ApiInternalOnlyEndpoint()
@@ -192,14 +140,13 @@ export class AuthController {
) {
this.logger.info('/auth - change-password');
const { oldPassword, newPassword, totpCode } = body;
const { oldPassword, newPassword } = body;
const { authorization: accessToken } = headers;
return this.authClient.changePassword({
accessToken,
oldPassword,
newPassword,
totpCode,
});
}
@@ -217,8 +164,7 @@ export class AuthController {
const { username } = body;
await this.authClient.resetPassword({ username }, metadata);
return { authProvider: process.env.AUTH_PROVIDER || 'cognito' };
return this.authClient.resetPassword({ username }, metadata);
}
@ApiInternalOnlyEndpoint()
@@ -474,59 +420,4 @@ export class AuthController {
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Check for API key header first
const apiKey = req.get('X-Api-key');
if (apiKey) {
this.logger.info('Authenticating via X-Api-key header');
const { api_key } = await this.apiKeyService.get(apiKey);
const userDto = {
id: api_key.user_id,
name: api_key.username,
customer: {
id: api_key.customer_id,
name: api_key.customer_name,
tier: api_key.customer_tier,
}
};
return res.status(200).json(userDto);
}
// Get token and headers
const accessToken = req.cookies['ddf-auth'];
const refreshToken = req.cookies['ddf-refresh-auth'];
const userId = req.cookies['ddf-user-id'];
const resourceHost = req.headers["host"]
const hasUserSession = Boolean(accessToken) && Boolean(userId);
this.logger.info('Has User Session: ' + hasUserSession);
if (!hasUserSession) {
throw new UnauthorizedException()
}
try {
const userDto = await this.authClient.validateUserSession(accessToken, resourceHost);
return res.status(200).json(userDto);
} catch (error) {
if (!refreshToken) {
this.logger.error('Invalid refresh token or customer name');
throw new UnauthorizedException("Invalid refresh token or customer name");
};
const {
authSession,
user
} = await this.authClient.refreshUserSession(refreshToken, userId, resourceHost);
this.authClient.writeAuthSession(res, authSession);
return res.status(200).json(user);
}
}
}
+1 -2
View File
@@ -8,11 +8,10 @@ import { AuthClientService } from './auth.service';
import { DucClient } from '../duc/client.config';
import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
import { getOauthSecrets } from 'src/utils/OauthSecrets';
import { ApiKeyModule } from '../api-key/api-key.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ApiKeyModule],
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AuthController],
providers: [
AuthClientService,
+8 -260
View File
@@ -1,20 +1,10 @@
import {
OnModuleInit,
Inject,
Injectable,
ForbiddenException,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import {
AuthProtoService as AuthServiceInterface,
UsersProtoService,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -27,28 +17,16 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse, UserDTO } from './dtos/login';
import jwt, { JwtPayload } from 'jsonwebtoken';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { Request, Response } from 'express';
type AuthSession = {
accessToken?: string;
refreshToken?: string;
userId?: string;
};
import { BulkEditResponse } from './dtos/login';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private userService: UsersProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -61,10 +39,6 @@ export class AuthClientService implements OnModuleInit {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
this.userService = this.grpcClient.getService<UsersProtoService>(
ProtoServices.UsersProtoService,
);
}
async getPublicKeys() {
@@ -79,63 +53,25 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({ customer }: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info(
'SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY,
);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info(
'Check if customer have network policy: ' + customer.modules,
);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result;
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
}
async refreshAccessToken(
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
{ refreshToken }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken(
{ refreshToken, userId },
metadata,
),
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
);
}
@@ -143,7 +79,6 @@ export class AuthClientService implements OnModuleInit {
accessToken,
oldPassword,
newPassword,
totpCode,
}: AuthChangePasswordRequest) {
this.logger.info('ChangePassword');
@@ -152,7 +87,6 @@ export class AuthClientService implements OnModuleInit {
accessToken,
oldPassword,
newPassword,
totpCode,
}),
);
}
@@ -305,190 +239,4 @@ export class AuthClientService implements OnModuleInit {
throw error;
}
}
public async validateUserSession(accessToken: any, resourceHost: string) {
const payload = await this.validateJwtToken(accessToken);
const userDto = await this.getUserfromPayload(payload);
this.validateResourceAccess(resourceHost, userDto);
return userDto;
}
public async refreshUserSession(
refreshToken: string,
userId: string,
originHeader: string,
): Promise<{
user: UserDTO;
authSession: AuthSession;
}> {
const metadata = PackTheMetadata({});
this.logger.info('Call Refresh Token');
const refreshCredentials = await this.refreshAccessToken(
{ refreshToken, userId },
metadata,
);
this.logger.info('Finish Refresh Token');
const userDto = await this.validateUserSession(
refreshCredentials.accessToken,
originHeader,
);
return {
user: userDto,
authSession: {
accessToken: refreshCredentials.accessToken,
refreshToken: refreshCredentials.refreshToken,
userId,
},
};
}
public writeAuthSession(res: Response, data: AuthSession) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth');
res.cookie('ddf-auth', data.accessToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth');
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth');
res.cookie('ddf-user-id', data.userId, {
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
public cleanUpAuthSession(res: Response) {
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.ai',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.ai',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
}
private async validateJwtToken(token: string) {
const decoded: any = token && jwt.decode(token, { complete: true });
if (!decoded) throw new Error('Invalid token');
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(token, pemValue);
return decoded.payload;
}
private async getUserfromPayload(payload: JwtPayload): Promise<UserDTO> {
this.logger.info('getUser');
const metadata = PackTheMetadata({
customer_id: payload.customer_id,
});
const { user } = await lastValueFrom(
this.userService.UserFindOneById({ id: payload.user_id }, metadata),
);
const userDto: UserDTO = {
id: user.id,
name: user.username,
jobTitle: user?.jobTitle || null,
department: user?.department || null,
hierarchy: user?.hierarchy || null,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
},
};
return userDto;
}
private validateResourceAccess(host: string, user: UserDTO) {
this.logger.info(
"Validate whether the source URL is a resource belonging to the user's client",
);
this.logger.info('Host: ' + host);
this.logger.info('Customer: ' + user.customer.name);
const hostParts = host.split('.');
const domain = hostParts[0];
const isResouceStg = hostParts[1] === 'stg';
const notFoundCustomerInDomain = !domain.includes('-')
if (notFoundCustomerInDomain) {
this.logger.info(`Not found Customer Name in domain`);
return;
}
const domainParts = domain.split('-');
const customerInDomain = domainParts[domainParts.length - 1];
if (isResouceStg && process.env.ENV !== 'stg') {
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
throw new HttpException(
`Customer ${user.customer.name} cannot access ${host}`,
HttpStatus.FORBIDDEN
);
}
if (customerInDomain != user.customer.name) {
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
throw new HttpException(
`Customer ${user.customer.name} cannot access ${host}`,
HttpStatus.FORBIDDEN
);
}
return;
}
}
+1 -14
View File
@@ -122,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
refreshToken: string;
@ApiProperty()
userId: string;
customerName: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
@@ -140,16 +140,3 @@ export interface BulkEditResponse {
successfulUsers: string[];
failedUsers: string[];
}
export type UserDTO = {
id: string,
name: string,
jobTitle?: string,
department?: string,
hierarchy?: string,
customer: {
id: string,
name: string,
tier: string,
}
}
@@ -26,7 +26,6 @@ export class GoogleLoginStrategy extends PassportStrategy(
callbackURL: oauthSecrets['google-login'].redirect_uri,
scope: ['email', 'profile', 'openid'],
};
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
const verify = (
accessToken: string,
refreshToken: string,
+7 -65
View File
@@ -32,7 +32,7 @@ import {
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
BatchRemoveRlsRulesRequest,
@@ -117,52 +117,6 @@ export class CatalogController {
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog/download - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
@@ -423,7 +377,6 @@ export class CatalogController {
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Query('asset_type') asset_type: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -440,7 +393,7 @@ export class CatalogController {
language,
});
const docs = await this.catalogService.getDataDocs(id, asset_type, metadata);
const docs = await this.catalogService.getDataDocs(id, metadata);
return { docs };
}
@@ -488,32 +441,21 @@ export class CatalogController {
@Headers() headers,
@Param('id') table_id: string,
@Body('docs') docs: string,
@Query('asset_type') asset_type: string,
) {
const { user_id, customer_name, customer_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const { user_id, customer_name } = user;
this.logger.info(`/catalog - ON POST DATA DOCS ROUTE`, {
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const body = {
const res = await this.catalogService.createDataDocs({
table_id,
docs,
asset_type,
info: {
customer: customer_name,
},
}
const res = await this.catalogService.createDataDocs(body, metadata);
});
return res;
}
@@ -943,4 +885,4 @@ export class CatalogController {
this.logger.error(error.message);
}
}
}
}
+1 -6
View File
@@ -3,15 +3,12 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@@ -21,8 +18,6 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
ShareModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
+55 -230
View File
@@ -24,11 +24,8 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
CreateDataDocsDTO,
IUpdateDataRequest,
TriggerCatalogReq,
} from './dtos';
@@ -38,6 +35,12 @@ import {
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { writeFileSync } from 'fs';
import path from 'path';
import { HtmlParser } from 'src/utils/FileParser/html-parser';
import { PiiDto } from './dtos/pii.dto';
import { CsvParser } from 'src/utils/FileParser/csv-parser';
import { PDFParser } from 'src/utils/FileParser/pdf-parser';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
@@ -87,23 +90,25 @@ class CatalogService implements OnModuleInit {
}
async getPiiReporter(metadata: Metadata, type: TypeParser) {
this.logger.info('getPiiReporter: ' + type);
this.logger.info('getPiiReporter: ' + type)
try {
const { data } = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata),
);
this.logger.info('Finish grpc call');
const {
data
} = await lastValueFrom(
this.catalogWriteService.GetPiiReporter({}, metadata)
)
this.logger.info("Finish grpc call")
const parser = ParserBuilder.build<PiiMetadata>(type);
this.logger.info('parser file to: ' + type);
const file = await parser.parse(data);
this.logger.info('finish parser');
this.logger.info('parser file to: ' + type)
const file = await parser.parse(data)
this.logger.info('finish parser')
const mimeTypes: Record<TypeParser, string> = {
csv: 'text/csv',
html: 'text/html',
pdf: 'application/pdf',
};
'csv': 'text/csv',
'html': 'text/html',
'pdf': 'application/pdf'
}
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `relatorio-pii-${timestamp}.${type}`;
@@ -111,12 +116,13 @@ class CatalogService implements OnModuleInit {
return {
file,
filename: filename,
type: mimeTypes[type],
};
type: mimeTypes[type]
}
} catch (error) {
this.logger.error(error.message);
throw error;
}
}
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
@@ -196,11 +202,9 @@ class CatalogService implements OnModuleInit {
async getUserRolesIds(userId: string) {
const result = await this.userService.findOneById(userId).catch(() => null);
if (result) {
return result.user.roles.map((role) => role.id);
}
const roles_ids = result.user.roles.map((role) => role.id);
return [];
return roles_ids;
}
async searchDataAssets(
@@ -208,99 +212,10 @@ class CatalogService implements OnModuleInit {
metadata: Metadata,
customer_id: string,
) {
this.logger.info('CatalogService - searchDataAssets', { query });
this.logger.info('CatalogService - searchDataAssets');
const { search, page, size, sort_by, order, ...filters } = query;
this.logger.debug('Extracted filters:', { filters });
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (
filters.manually !== undefined &&
filters.manually !== null &&
filters.manually !== ''
) {
filters.manually = Number(filters.manually); // 1 ou 0
} else {
delete filters.manually;
}
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
search,
page,
size,
sort_by,
order,
filters,
});
if (filters.owner) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
this.logger.info('Available users in database count:', {
count: customer_users.length,
});
this.logger.info('First 5 users:', {
users: customer_users
.slice(0, 5)
.map((u) => ({ id: u.id, email: u.email, name: u.name })),
});
const ownerValues = Array.isArray(filters.owner)
? filters.owner
: typeof filters.owner === 'string' && filters.owner.includes(',')
? filters.owner.split(',').map((o: string) => o.trim())
: [filters.owner];
this.logger.info('Owner values to convert:', {
ownerValues,
ownerFiltersOriginal: filters.owner,
});
const ownerIds = ownerValues
.map((ownerValue: string) => {
const normalizedOwner = ownerValue.replace(/\s/g, '+');
const user = customer_users.find((u) => {
const isIdMatch = u.id === ownerValue;
const isEmailMatch =
u.email === ownerValue || u.email === normalizedOwner;
const isNameMatch =
u.name === ownerValue || u.name === normalizedOwner;
this.logger.info('Comparing:', {
userId: u.id,
userEmail: u.email,
userName: u.name,
filterValue: ownerValue,
normalizedFilter: normalizedOwner,
idMatch: isIdMatch,
emailMatch: isEmailMatch,
nameMatch: isNameMatch,
});
return isIdMatch || isEmailMatch || isNameMatch;
});
this.logger.info('Looking for owner result:', {
ownerValue,
found: !!user,
userId: user?.id,
});
return user?.id || ownerValue;
})
.filter((id: string) => id);
if (ownerIds.length > 0) {
filters.owner = ownerIds;
}
}
const { data_assets, total } = await lastValueFrom(
this.catalogReadService.GetAllDataAssets(
{
@@ -315,8 +230,6 @@ class CatalogService implements OnModuleInit {
),
);
console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
const result = JSON.parse(data_assets);
const response = await this.getAssetsUsersAndRoles(
@@ -327,34 +240,6 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map((asset) => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']',
}));
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename,
};
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -427,11 +312,11 @@ class CatalogService implements OnModuleInit {
return { data_asset: asset[0] };
}
async getDataDocs(id: string, assetType: string, metadata: Metadata) {
async getDataDocs(id: string, metadata: Metadata) {
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id }, metadata),
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
@@ -458,16 +343,7 @@ class CatalogService implements OnModuleInit {
return result;
}
async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
if (body.asset_type === 'table' || body.asset_type === 'view') {
return this.createDataDocsViaNimbus(body);
}
return this.createDataDocsViaGrpc(body, metadata);
}
private async createDataDocsViaNimbus(body: CreateDataDocsDTO) {
this.logger.info('Creating data docs via Nimbus for table/view');
async createDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
`${nimbusUrl}/api/catalog/data-docs/`,
@@ -476,29 +352,6 @@ class CatalogService implements OnModuleInit {
return data;
}
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
this.logger.info('Creating data docs via gRPC for other asset types');
try {
const response: any = await lastValueFrom(
this.catalogWriteService.UpdateDataAssetDoc(
{
id: body.table_id,
docs: body.docs,
},
metadata,
),
);
return response;
} catch (error) {
this.logger.error('Error creating data asset docs:', error);
throw new HttpException(
'Failed to create data asset documentation',
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
@@ -516,7 +369,6 @@ class CatalogService implements OnModuleInit {
return response;
}
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
@@ -527,20 +379,17 @@ class CatalogService implements OnModuleInit {
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.email;
)?.username;
const roles = [];
const users = [];
const data_asset_roles = data_asset?.roles || [];
for (const role_id of data_asset_roles) {
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
const data_asset_users = data_asset?.users || [];
for (const user_id of data_asset_users) {
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, email: user.email });
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
@@ -630,36 +479,27 @@ class CatalogService implements OnModuleInit {
async createTableMetadata(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
this.logger.info(
`Creating table metadata for table ${body.table_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {
...body.logMetadata,
});
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, {
...body.table_metadata,
});
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
this.logger.info(
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
return data.id;
} catch (error) {
this.logger.error(
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
error.response?.status
} because of ${JSON.stringify(error.response?.data) || error.message}`,
{ ...body.logMetadata },
);
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
@@ -669,58 +509,43 @@ class CatalogService implements OnModuleInit {
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
try {
this.logger.info(
`Creating column metadata for table ${body.column_metadata.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.column_metadata)}`,
{ ...body.logMetadata },
);
const { data, status } = await axios.post(
endpoint,
body.column_metadata,
);
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
const { data, status } = await axios.post(endpoint, body.column_metadata);
this.logger.info(
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
return data.map((column) => column.id);
} catch (error) {
this.logger.error(
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{ ...body.logMetadata },
);
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
throw new Error(error.response?.data?.message || error.message);
}
}
async createDataPreview(body: any): Promise<number> {
const nimbusUrl = this._getNimbusUrl(body);
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
this.logger.info(
`Creating data preview for table ${body.data_preview.table_name}`,
{ ...body.logMetadata },
);
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
this.logger.debug(
`Payload: ${JSON.stringify(body.data_preview)}`,
{ ...body.logMetadata },
);
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
try {
const { data, status } = await axios.post(endpoint, body.data_preview);
this.logger.info(
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
return data.id;
} catch (error) {
@@ -728,7 +553,7 @@ class CatalogService implements OnModuleInit {
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
error.response?.status
} because of ${error.response?.data || error.message}`,
{ ...body.logMetadata },
{...body.logMetadata},
);
throw new Error(error.response?.data?.message || error.message);
}
-39
View File
@@ -98,8 +98,6 @@ export class IDataAsset {
embed?: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class IOneDataAsset {
@@ -149,24 +147,6 @@ export class ICatalogAllRequest {
description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
})
order?: OrderEnum;
@ApiPropertyOptional({
description: 'ID do usuário owner para filtrar data assets',
example: 'user-id-1,user-id-2',
})
owner?: string;
@ApiPropertyOptional({
description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)',
example: '2025-01-01',
})
catalog_date_from?: string;
@ApiPropertyOptional({
description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)',
example: '2025-12-31',
})
catalog_date_to?: string;
}
export class ICatalogAllResponse {
@@ -202,8 +182,6 @@ export class IUpdateDataRequest {
embed: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
@ApiPropertyOptional()
docs?: string;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
@@ -218,8 +196,6 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
location: string;
@ApiPropertyOptional()
embed: EmbedObject;
@ApiPropertyOptional()
docs: string;
}
export class IPreview {
@@ -344,18 +320,3 @@ export class BatchRemoveRlsRulesRequest {
@ApiPropertyOptional()
id_rls?: string;
}
export type AssetReporter = {
id: string;
display_name: string;
data_asset_type: string;
created_at: string;
tags: string;
}
export type CreateDataDocsDTO = {
table_id: string;
docs: string;
asset_type: string;
}
@@ -1,89 +0,0 @@
import {
Controller,
Get,
Inject,
Param,
Req,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ShareService } from './share.service';
import { Request } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog/data-asset/share')
@UseFilters(new GrpcToHttpExceptionFilter())
export class ShareController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogShareService: ShareService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
}
@Get('/:id/preview')
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@Get('/:id/docs')
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
}
-30
View File
@@ -1,30 +0,0 @@
import { Module } from "@nestjs/common";
import { CatalogClientConfiguration } from "../catalog-client";
import { ClientsModule } from "@nestjs/microservices";
import { RolesModule } from "src/modules/roles/roles.module";
import { UsersModule } from "src/modules/users/users.module";
import { CustomersModule } from "src/modules/customers/customers.module";
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
UsersModule,
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
exports: [ShareModule],
})
export class ShareModule {}
-275
View File
@@ -1,275 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
ProtoServices,
ReadService,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
ForbiddenException,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { CatalogClientConfiguration } from '../catalog-client';
import { ClientGrpc } from '@nestjs/microservices';
import { UsersService } from 'src/modules/users/users.service';
import { RolesService } from 'src/modules/roles/roles.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id
}))
try {
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
type: isJWT(id) ? 'assigned' : shareMetadata.type,
customer: shareMetadata.customerName
}
if (user) {
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
} else {
await this.mixpanelService.trackShare(request, mixpanelTracker);
}
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { data_asset } = await this.getOneDataAsset({
customer_id: shareMetadata.customerId,
id: shareMetadata.assetId,
metadata,
});
this.logger.info('found asset: ' + JSON.stringify(data_asset));
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public') throw new NotFoundException();
return { data_asset };
} catch (error) {
this.logger.error(error);
throw error;
}
}
private async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: undefined },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
private async getAssetsUsersAndRoles(
data_assets: Array<any>,
customer_id: string,
) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.email;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, email: user.email });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
if (isJWT(id) && info ){
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return info;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -20,7 +20,7 @@ import {
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@Injectable()
export class ConnectionTestService {
@@ -28,7 +28,7 @@ import {
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
+1 -29
View File
@@ -25,7 +25,7 @@ import { CustomersService } from './customers.service';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { EnforceMfa } from './dtos/enforce-mfa';
@ApiTags('Customers')
@@ -136,32 +136,4 @@ export class CustomersController {
const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
return result;
}
@Get(':id/organization-info')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ description: 'Organization information' })
async getOrganizationInfo(@Param('id') id: string) {
this.logger.info('getOrganizationInfo', { id });
return this.customersService.getOrganizationInfo(id);
}
@Put(':id/organization-info')
@Authenticated()
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ description: 'Organization information updated' })
async updateOrganizationInfo(
@Param('id') id: string,
@Body() body: {
companyName: string;
companySite: string;
domain: string;
cnpj: string;
description: string;
},
) {
return this.customersService.updateOrganizationInfo(id, body);
}
}
+1 -64
View File
@@ -59,14 +59,6 @@ export class CustomersService implements OnModuleInit {
);
}
async getCustomer(customerId: string) {
return await lastValueFrom(
this.customerService.CustomerFindOneById({
id: customerId
})
)
}
async getLinks(customerId: string) {
try {
const result = await lastValueFrom(
@@ -149,7 +141,6 @@ export class CustomersService implements OnModuleInit {
// const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token;
}
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
@@ -167,7 +158,6 @@ export class CustomersService implements OnModuleInit {
return res;
}
async getLogsDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
@@ -223,57 +213,4 @@ export class CustomersService implements OnModuleInit {
})
)
}
async updateOrganizationInfo(
customerId: string,
data: {
companyName: string;
companySite: string;
domain: string;
cnpj: string;
description: string;
},
) {
try {
const result = await lastValueFrom(
this.customerService.OrganizationUpdate({
customerId,
companyName: data.companyName || '',
companySite: data.companySite || '',
domain: data.domain || '',
cnpj: data.cnpj || '',
description: data.description || '',
}),
);
return result;
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
async getOrganizationInfo(customerId: string) {
try {
const customerResponse = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId })
);
const customer = customerResponse.customer;
return {
companyName: customer.companyName || '',
companySite: customer.companySite || '',
domain: customer.domain || '',
cnpj: customer.cnpj || '',
description: customer.description || ''
};
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
}
}
@@ -1,27 +0,0 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
export class OrganizationUpdateRequest {
@ApiProperty()
name: string;
@ApiPropertyOptional()
companySite: string;
@ApiProperty()
domain: string;
@ApiPropertyOptional()
info: string;
@ApiPropertyOptional()
cnpj: string;
}
export class OrganizationResponse {
@ApiProperty()
name: string;
@ApiPropertyOptional()
companySite: string;
@ApiProperty()
domain: string;
@ApiPropertyOptional()
info: string;
@ApiPropertyOptional()
cnpj: string;
}
@@ -1,47 +0,0 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateIdentityProvider {
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
clientSecret: string;
@ApiProperty()
issuerUrl: string;
@ApiProperty()
permissions: number[];
}
export class IdentityProviderResponse {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
issueUrl: string;
@ApiProperty()
permissions: {
id: number;
name: string;
}[];
}
export class IdentityProviderListResponse {
@ApiProperty()
providers: IdentityProviderResponse[]
}
@@ -1,10 +0,0 @@
export class SsoSignInDto {
readonly nonce: string;
readonly codeVerifier: string;
readonly state: string;
readonly id: string;
readonly clientId: string;
readonly clientSecret: string;
readonly issuerUrl: string;
readonly redirectUrls: string[];
}
@@ -1,246 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Redirect,
Req,
} from '@nestjs/common';
import { IdentityProviderService } from './identity-provider.service';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiOkResponse } from '@nestjs/swagger';
import {
CreateIdentityProvider,
IdentityProviderListResponse,
IdentityProviderResponse,
} from './dto/identity-provider.dto';
import { Request } from 'express';
import ErrorCodes from 'src/utils/errorCodes';
import {
Authenticated,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('identity-providers')
export class IdentityProviderController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private identityProviderService: IdentityProviderService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async addIdentityProvider(
@User() user: RequestUser,
@Body() body: CreateIdentityProvider,
@Language() language: LanguageEnum,
) {
this.logger.info('POST /identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
return await this.identityProviderService.create(body, metadata);
}
@Get()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderListResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async getProviders(
@User() user: RequestUser,
@Language() language: LanguageEnum,
) {
this.logger.info('GET identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
const result = await this.identityProviderService.getList(metadata);
return result;
}
@Delete(':id')
@HttpCode(HttpStatus.NO_CONTENT)
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async deleteIdentityProvider(
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('DELETE /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.deleteIdentityProvider(
id,
metadata,
);
}
@Put(':id')
@HttpCode(HttpStatus.OK)
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async updateIdentityProviders(
@Param('id') id: string,
@Body() body: CreateIdentityProvider,
@User() user: RequestUser,
) {
this.logger.info('PUT /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.updateIdentityProviders(
id,
body,
metadata,
);
}
@Post('/callback')
@HttpCode(HttpStatus.OK)
async callbackIdp(
@Req() req: Request,
@Language() language: LanguageEnum,
@Body()
body: {
state: string;
code: string;
},
) {
this.logger.info('GET /identity-providers/callback');
const { code, state } = body;
if (!code) {
this.logger.error('No code received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
if (!state) {
this.logger.error('No state received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@HttpCode(HttpStatus.OK)
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -1,16 +0,0 @@
import { Module } from '@nestjs/common';
import { IdentityProviderController } from './identity-provider.controller';
import { IdentityProviderService } from './identity-provider.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ServicesModule } from 'src/services/service.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
controllers: [IdentityProviderController],
providers: [IdentityProviderService, DadosferaLogger],
})
export class IdentityProviderModule {}
@@ -1,185 +0,0 @@
import {
BadRequestException,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
client_secret: idp.clientSecret,
redirect_uris: idp.redirectUrls,
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
id: idp.id,
state,
clientId: idp.clientId,
clientSecret: idp.clientSecret,
issuerUrl: idp.issuerUrl,
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
code_challenge,
code_challenge_method: 'S256',
state,
nonce,
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
client_secret: ssoSign.clientSecret,
redirect_uris: ssoSign.redirectUrls,
});
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
idToken: tokenSet.id_token,
refreshToken: tokenSet.refresh_token,
id: ssoSign.id,
}),
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
}
async updateIdentityProviders(
id: string,
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
id,
...body,
},
metadata,
),
);
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
}
}
+79 -24
View File
@@ -1,46 +1,101 @@
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
import { extractUserFrom } from 'src/authentication/extract-user';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private mixpanelService: MixpanelService,
) {
this.logger = dadosferaLogger.logger;
}
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
) {}
@Post(':id')
async trackEvent(
@Param('id') id,
@Body() body,
@User() user: RequestUser,
@Req() request
) {
this.logger.info(`POST Track Event: ${id}`)
delete body.info;
const mixpanel = init(this.mixpanelToken);
const anonymousUser = {
username: "anonymous",
customer_name: "anonymous"
} as RequestUser
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
const hasToken = request.headers['authorization'];
let username = user.username;
const user = hasToken ? extractUserFrom(hasToken) : anonymousUser;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
this.logger.info(`Has user: ${typeof hasToken == "string"}`)
await this.mixpanelService.track(id, user, request, body)
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
this.logger.info(`Event successful`)
return { id, body, user: user.username };
}
capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
+1 -6
View File
@@ -1,8 +1,6 @@
import { Module } from '@nestjs/common';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
import { MixpanelService } from './mixpanel.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [MixpanelController],
@@ -10,12 +8,9 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs';
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecretFromSecretsManager(
`prd/root/mixpanel_token`,
`${process.env.ENV}/root/mixpanel_token`,
),
},
MixpanelService,
DadosferaLogger
],
exports: [MixpanelService]
})
export class MixpanelModule {}
-118
View File
@@ -1,118 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Inject } from '@nestjs/common';
import { Request } from 'express';
import mixpanel, { init } from 'mixpanel';
import { RequestUser } from 'src/decorators/user.decorator';
export class MixpanelService {
logger: DadosferaLogger;
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async track(eventName: string, user: RequestUser, request: Request, body: any) {
this.logger.info("track: " + JSON.stringify({
eventName,
...body
}))
const mixpanel = init(this.mixpanelToken);
await this.setPeople(user, mixpanel);
await mixpanel.track(eventName, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
async trackShare(request: Request, body: any) {
this.logger.info("trackShare: " + JSON.stringify(body))
const mixpanel = init(this.mixpanelToken);
await mixpanel.track("share_page", {
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
}
private capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
private getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
@@ -8,7 +8,7 @@ import {
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { NetworkPolicyService } from './network-policy.service';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@Controller('network-policy')
export class NetworkPolicyController {
+1 -1
View File
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
@@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator';
import { OpenDataService } from './open-data.service';
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { request } from 'http';
import { Request } from 'express';
@@ -45,11 +45,11 @@ export class OpenDataController {
) {
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
// if (!corslist.includes(origin)) {
// this.logger.info('block request by cors list: '+ origin);
// throw new ForbiddenException();
// }
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
if (!corslist.includes(origin)) {
this.logger.info('block request by cors list: '+ origin);
throw new ForbiddenException();
}
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
+20 -3
View File
@@ -3,7 +3,6 @@ import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
@@ -14,6 +13,26 @@ import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
);
})
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -25,7 +44,6 @@ export class PipelinesController {
}
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiOperation({
deprecated: true,
description:
@@ -53,7 +71,6 @@ export class PipelinesController {
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
+35 -22
View File
@@ -26,14 +26,13 @@ import {
import {
AuthenticateCondition,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
@@ -53,6 +52,30 @@ import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2')
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
case 'PATCH':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
);
})
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -65,7 +88,6 @@ export class PipelinesController {
}
@Get('monitoring-dashboard')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getMonitoringDashboard(@User() user: RequestUser) {
this.logger.info('PipelinesController - getMonitoringDashboard', { user });
@@ -78,7 +100,6 @@ export class PipelinesController {
}
@Post()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiCreatedResponse({ type: IPipelineV2 })
async create(
@Language() language: LanguageEnum,
@@ -105,7 +126,6 @@ export class PipelinesController {
}
@Get()
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async findAll(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -130,7 +150,6 @@ export class PipelinesController {
}
@Get('/download-logs')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async downloadLogs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -161,7 +180,6 @@ export class PipelinesController {
}
@Get(':id/config')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW,PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineproperties(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -173,7 +191,6 @@ export class PipelinesController {
}
@Get(':id/objects')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineObjects(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -185,14 +202,16 @@ export class PipelinesController {
}
@Get(':id/status')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.oldPipelinesService.getPipelineStatus(body);
@@ -200,7 +219,6 @@ export class PipelinesController {
}
@Get('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async findOne(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -238,12 +256,10 @@ export class PipelinesController {
});
return res;
});
return result;
}
@Patch('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async update(
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@@ -283,7 +299,6 @@ export class PipelinesController {
deprecated: true,
description: 'This method is deprecated. Please use PATCH instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async updateDeprecated(
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@@ -299,7 +314,6 @@ export class PipelinesController {
@Delete(':id')
@ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT)
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
async delete(@Param('id') id: string, @User() user: RequestUser) {
this.logger.info('PipelinesController - delete', { user });
const metadata = PackTheMetadata({
@@ -313,7 +327,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/init-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async initUploadFile(
@User() user: RequestUser,
@Body() body: IInitUploadCSVFile,
@@ -345,7 +359,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/complete-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async completeUploadFile(
@User() user: RequestUser,
@Body() body: ICompleteUploadCSVFile,
@@ -363,7 +377,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/file')
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async uploadedFile(
@User() user: RequestUser,
@Body() body: ICreatePipelineCSVFile,
@@ -413,7 +427,6 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
@@ -1,11 +0,0 @@
export const PLATFORM_API_CONFIG = {
getUrl: (): string => {
const url = process.env.PLATFORM_API_URL;
if (!url) {
throw new Error('PLATFORM_API_URL environment variable is not set');
}
return url;
},
region: process.env.AWS_REGION || 'us-east-1',
timeout: parseInt(process.env.PLATFORM_API_TIMEOUT || '30000', 10),
};
File diff suppressed because it is too large Load Diff
@@ -1,17 +0,0 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PlatformApiController } from './platform-api.controller';
import { PlatformApiService } from './platform-api.service';
import { ElasticsearchModule } from '../../services/elasticsearch';
import { DynamoDBModule } from '../../services/dynamodb';
import { CustomersModule } from '../customers/customers.module';
@Module({
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule],
controllers: [PlatformApiController],
providers: [PlatformApiService, DadosferaLogger],
exports: [PlatformApiService],
})
export class PlatformApiModule {}
@@ -1,123 +0,0 @@
import { Injectable, Inject, HttpException } from '@nestjs/common';
import { SignatureV4 } from '@aws-sdk/signature-v4';
import { Sha256 } from '@aws-crypto/sha256-js';
import { defaultProvider } from '@aws-sdk/credential-provider-node';
import axios, { AxiosResponse, Method } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser } from '../../decorators/user.decorator';
import { PLATFORM_API_CONFIG } from './platform-api.config';
@Injectable()
export class PlatformApiService {
private signer: SignatureV4;
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
this.signer = new SignatureV4({
service: 'execute-api',
region: PLATFORM_API_CONFIG.region,
credentials: defaultProvider(),
sha256: Sha256,
});
}
async proxy(
method: string,
path: string,
user: RequestUser,
body?: any,
query?: Record<string, string>,
): Promise<any> {
const baseUrl = PLATFORM_API_CONFIG.getUrl();
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
host: url.hostname,
'content-type': 'application/json',
// Forward user context headers
// Note: platform-api expects customer_name in the 'customer_id' header (contract inconsistency)
'customer_id': user.customer_name || '',
'customer_name': user.customer_name || '',
'x-user-id': user.user_id || '',
'x-username': user.username || '',
'x-customer-tier': user.customer_tier || '',
'x-customer-id': user.customer_id || '',
};
const requestToSign = {
method: method.toUpperCase(),
protocol: url.protocol,
hostname: url.hostname,
port: url.port ? parseInt(url.port, 10) : undefined,
path: url.pathname + url.search,
headers,
body: body ? JSON.stringify(body) : undefined,
};
this.logger.info('Proxying request to platform-api', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
user_id: user.user_id,
});
try {
// Sign with IAM v4
const signedRequest = await this.signer.sign(requestToSign);
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers: signedRequest.headers as Record<string, string>,
data: body,
timeout: PLATFORM_API_CONFIG.timeout,
validateStatus: () => true, // Don't throw on non-2xx
});
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Platform API proxy error', {
error: error.message,
status: error.response?.status,
path,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
if (error.code === 'ECONNREFUSED') {
throw new HttpException('Platform API service unavailable', 503);
}
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
throw new HttpException('Platform API request timeout', 504);
}
throw new HttpException('Internal server error', 500);
}
}
}
+31 -22
View File
@@ -20,6 +20,7 @@ import { PermissionsService } from '../permissions/permissions.service';
import { LanguageEnum } from 'src/utils/languages.enum';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { PUBLIC_PERMISSIONS_SEQID } from 'src/authentication/permissions.enum';
interface GetRolesPermissionsName {
id: string;
@@ -103,6 +104,20 @@ export class RolesService {
const meta = new Metadata();
meta.add('access_token', access_token);
const { description, name, permissionIds, userIds } = data;
const invalidPermissions = [];
for (const permission of permissionIds) {
if (!PUBLIC_PERMISSIONS_SEQID.includes(permission)) {
invalidPermissions.push(permission);
}
}
if (invalidPermissions.length > 0) {
throw new BadRequestException(
`Invalid permissions: ${invalidPermissions.join(', ')}`,
);
}
const role = await lastValueFrom(
this.rolesClientService.RoleCreate(
{
@@ -229,33 +244,27 @@ export class RolesService {
const [roleTreated] = this.getRolesPermissionsName([role.role]);
return { role: roleTreated };
}
getRolesPermissionsName(roles: GetRolesPermissionsName[]): RoleDto[] {
const newRoles: RoleDto[] = [];
for (const role of roles) {
const newRole: RoleDto = this.formatRole(role);
const allPermissions = this.permissionsService.getAllPermissions(
this.language,
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
const newRole: RoleDto = {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};
newRoles.push(newRole);
}
return newRoles;
}
private formatRole(role: GetRolesPermissionsName): RoleDto {
const allPermissions = this.permissionsService.getAllPermissions(
this.language
);
const newPermissions = role.permissions.map((p) => {
const permission = allPermissions.find((per) => per.seqid === p.seqid);
return {
...p,
name: permission.name,
id: p.seqid,
};
});
return {
...role,
permissions: newPermissions,
isPublic: role.isPublic,
};;
}
}
@@ -1,10 +0,0 @@
export class CreateShareMetadataDto {
assetId: string;
proposedId: string;
type: string;
}
export class UpdateShareMetadataDto {
shareId: string;
type: string;
}
@@ -1,31 +0,0 @@
import { Controller, Get, Body, Param, Put, Patch } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Controller('share')
@Authenticated()
export class ShareMetadataController {
constructor(private readonly shareMetadataService: ShareMetadataService) {}
@Put('/')
findOrcreate(@Body() data: CreateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.create(data, metadata);
}
@Patch('/')
updateShareType(@Body() data: UpdateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.updateShare(data, metadata);
}
@Get('/:id')
get(@Param('id') id: string, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.get(id, metadata);
}
}
@@ -1,16 +0,0 @@
import { Module } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { ShareMetadataController } from './share-metadata.controller';
import { DucClient } from '../duc/client.config';
import { ClientsModule } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [ShareMetadataController],
providers: [ShareMetadataService, DadosferaLogger],
exports: [ShareMetadataService]
})
export class ShareMetadataModule {}
@@ -1,62 +0,0 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Metadata } from '@grpc/grpc-js';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ShareMetadataProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Injectable()
export class ShareMetadataService implements OnModuleInit {
ducService: ShareMetadataProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<ShareMetadataProtoService>(
ProtoServices.ShareMetadataProtoService,
);
}
create(createShareDto: CreateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.FindOrCreateShareMetadata(
{
assetId: createShareDto.assetId,
proposedId: createShareDto.proposedId,
type: createShareDto.type
},
metadata,
),
);
}
updateShare(updateShare: UpdateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.ChangeShareMetadataType(
updateShare,
metadata,
),
);
}
get(id: string, metadata: Metadata) {
return lastValueFrom(
this.ducService.GetShareMetadata(
{
id,
},
metadata,
),
);
}
}
@@ -1,13 +0,0 @@
export const STORAGE_EXPLORER_CONFIG = {
getUrl: (customerName: string): string => {
const urlTemplate = process.env.STORAGE_EXPLORER_API_URL;
if (!urlTemplate) {
throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set');
}
// Replace {customer_id} placeholder with actual customer ID
// For local: http://172.17.0.1:8000/api (no placeholder)
// For prod: https://storage-explorer-{customer_id}.dadosfera.ai/api
return urlTemplate.replace('{customer}', customerName);
},
timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10),
};
@@ -1,383 +0,0 @@
import {
Controller,
Get,
Post,
Put,
Param,
Body,
Query,
Inject,
UseInterceptors,
UploadedFiles,
Headers,
} from '@nestjs/common';
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
import { FilesInterceptor } from '@nestjs/platform-express';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import FormData from 'form-data';
import {
Authenticated,
RequireAllPermissions,
} from '../../decorators/authentication.decorator';
import { User, RequestUser } from '../../decorators/user.decorator';
import { StorageExplorerService } from './storage-explorer.service';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
@ApiTags('Storage Explorer')
@Controller('storage-explorer')
@Authenticated()
export class StorageExplorerController {
private logger: any;
constructor(
private readonly storageExplorerService: StorageExplorerService,
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
// ============================================
// TABLE OPERATIONS
// ============================================
@ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' })
@Post('tables/validate-name')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async validateTableName(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/validate-name',
user,
body,
);
}
@ApiOperation({ summary: 'Create a new table' })
@Post('tables')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async createTable(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/tables/',
user,
body,
);
}
@ApiOperation({ summary: 'List all tables with pagination' })
@Get('tables')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listTables(
@Query('page') page: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/tables/',
user,
undefined,
{ page },
);
}
@ApiOperation({ summary: 'Get table details by ID' })
@Get('tables/:tableId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTable(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}`,
user,
);
}
@ApiOperation({ summary: 'Link a dataset to a table' })
@Post('tables/:tableId/datasets/:datasetId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async linkDatasetToTable(
@Param('tableId') tableId: string,
@Param('datasetId') datasetId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
`/tables/${tableId}/datasets/${datasetId}`,
user,
);
}
@ApiOperation({ summary: 'Get all datasets linked to a table' })
@Get('tables/:tableId/datasets')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTableDatasets(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/datasets`,
user,
);
}
@ApiOperation({ summary: 'Get table schema' })
@Get('tables/:tableId/schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getTableSchema(
@Param('tableId') tableId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/tables/${tableId}/schema`,
user,
);
}
@ApiOperation({ summary: 'Validate schema compatibility between table and dataset' })
@Post('tables/:tableId/validate-compatibility/:datasetId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async validateSchemaCompatibility(
@Param('tableId') tableId: string,
@Param('datasetId') datasetId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
`/tables/${tableId}/validate-compatibility/${datasetId}`,
user,
);
}
// ============================================
// DATASET OPERATIONS
// ============================================
@ApiOperation({ summary: 'Get dataset preview data' })
@Get('datasets/:datasetId/preview')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getDatasetPreview(
@Param('datasetId') datasetId: string,
@Query('limit') limit: number,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/preview`,
user,
undefined,
{ limit },
);
}
@ApiOperation({ summary: 'Get dataset schema information' })
@Get('datasets/:datasetId/schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getDatasetSchema(
@Param('datasetId') datasetId: string,
@Query('force_refresh') forceRefresh: boolean,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/${datasetId}/schema`,
user,
undefined,
{ force_refresh: forceRefresh },
);
}
@ApiOperation({ summary: 'List all datasets for a specific upload' })
@Get('datasets/upload/:uploadId')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listDatasetsByUpload(
@Param('uploadId') uploadId: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
`/datasets/upload/${uploadId}`,
user,
);
}
@ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' })
@Put('datasets/:datasetId/refresh-schema')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async refreshDatasetSchema(
@Param('datasetId') datasetId: string,
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'PUT',
`/datasets/${datasetId}/refresh-schema`,
user,
body,
);
}
// ============================================
// STORAGE OPERATIONS
// ============================================
@ApiOperation({ summary: 'List file explorer uploads with pagination' })
@Get('storage/uploads/history')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async listFileExplorerUploads(
@Query('page') page: number,
@Query('limit') limit: number,
@Query('folder_path') folderPath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/uploads/history',
user,
undefined,
{ page, limit, folder_path: folderPath },
);
}
@ApiOperation({ summary: 'Browse folders and files in storage' })
@Get('storage/browse')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async browseStorage(
@Query('path') path: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/browse',
user,
undefined,
{ path },
);
}
@ApiOperation({ summary: 'Upload multiple files to storage' })
@Post('storage/upload/batch')
@ApiConsumes('multipart/form-data')
@UseInterceptors(FilesInterceptor('files'))
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async batchUpload(
@UploadedFiles() files: Array<Express.Multer.File>,
@Body('folder_path') folderPath: string,
@User() user: RequestUser,
) {
// Create FormData to forward files to storage-explorer API
const formData = new FormData();
// Add files
if (files && files.length > 0) {
files.forEach((file) => {
formData.append('files', file.buffer, {
filename: file.originalname,
contentType: file.mimetype,
});
});
}
// Add folder_path
if (folderPath) {
formData.append('folder_path', folderPath);
}
return this.storageExplorerService.proxyFormData(
'POST',
'/storage/upload/batch',
user,
formData,
);
}
@ApiOperation({ summary: 'Create a new folder in storage' })
@Post('storage/folder/create')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE)
async createFolder(
@Body() body: any,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'POST',
'/storage/folder/create',
user,
body,
);
}
@ApiOperation({ summary: 'Download a file from storage' })
@Get('storage/download')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async downloadFile(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/download',
user,
undefined,
{ file_path: filePath },
);
}
@ApiOperation({ summary: 'Get detailed file metadata' })
@Get('storage/metadata')
@RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ)
async getFileMetadata(
@Query('file_path') filePath: string,
@User() user: RequestUser,
) {
return this.storageExplorerService.proxy(
'GET',
'/storage/metadata',
user,
undefined,
{ file_path: filePath },
);
}
}
@@ -1,13 +0,0 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { StorageExplorerController } from './storage-explorer.controller';
import { StorageExplorerService } from './storage-explorer.service';
@Module({
imports: [],
controllers: [StorageExplorerController],
providers: [StorageExplorerService, DadosferaLogger],
exports: [StorageExplorerService],
})
export class StorageExplorerModule {}
@@ -1,178 +0,0 @@
import { Injectable, Inject, HttpException } from '@nestjs/common';
import axios, { AxiosResponse, Method } from 'axios';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser } from '../../decorators/user.decorator';
import { STORAGE_EXPLORER_CONFIG } from './storage-explorer.config';
@Injectable()
export class StorageExplorerService {
private logger: any;
constructor(
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async proxy(
method: string,
path: string,
user: RequestUser,
body?: any,
query?: Record<string, any>
): Promise<any> {
// Validate customer_id is present for multi-tenant isolation
if (!user.customer_id) {
throw new HttpException('Customer ID is required for storage operations', 400);
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
'content-type': 'application/json',
};
this.logger.info('Proxying request to storage-explorer', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
storage_url: baseUrl,
user_id: user.user_id,
});
try {
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers,
data: body,
timeout: STORAGE_EXPLORER_CONFIG.timeout,
validateStatus: () => true, // Don't throw on non-2xx
});
// Propagate non-2xx responses as HttpExceptions
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Storage Explorer API proxy error', {
error: error.message,
status: error.response?.status,
path,
storage_url: baseUrl,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
if (error.code === 'ECONNREFUSED') {
throw new HttpException('Storage Explorer API service unavailable', 503);
}
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
throw new HttpException('Storage Explorer API request timeout', 504);
}
throw new HttpException('Internal server error', 500);
}
}
/**
* Proxy with file upload support (multipart/form-data)
*/
async proxyFormData(
method: string,
path: string,
user: RequestUser,
formData: any,
query?: Record<string, any>,
): Promise<any> {
// Validate customer_id is present for multi-tenant isolation
if (!user.customer_id) {
throw new HttpException('Customer ID is required for storage operations', 400);
}
// Get customer-specific storage-explorer URL
const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_name);
const url = new URL(`${baseUrl}${path}`);
// Add query params
if (query) {
Object.entries(query).forEach(([key, value]) => {
if (value !== undefined && value !== null) {
url.searchParams.set(key, String(value));
}
});
}
const headers: Record<string, string> = {
// Let axios set Content-Type for multipart/form-data with boundary
...formData.getHeaders?.(),
};
this.logger.info('Proxying form data request to storage-explorer', {
method: method.toUpperCase(),
path,
customer_id: user.customer_id,
storage_url: baseUrl,
user_id: user.user_id,
});
try {
const response: AxiosResponse = await axios({
method: method as Method,
url: url.href,
headers,
data: formData,
timeout: STORAGE_EXPLORER_CONFIG.timeout,
maxContentLength: Infinity,
maxBodyLength: Infinity,
validateStatus: () => true,
});
if (response.status >= 400) {
throw new HttpException(response.data, response.status);
}
return response.data;
} catch (error) {
this.logger.error('Storage Explorer API form data proxy error', {
error: error.message,
status: error.response?.status,
path,
storage_url: baseUrl,
method: method.toUpperCase(),
});
if (error instanceof HttpException) {
throw error;
}
if (error.response) {
throw new HttpException(error.response.data, error.response.status);
}
throw new HttpException('Internal server error', 500);
}
}
}
-20
View File
@@ -124,24 +124,4 @@ export class ThemeController {
}
}
@Post('/:id/theme/reset')
@ApiOkResponse({ type: CustomerThemeResponse })
async resetTheme(@Param('id') id: string) {
this.logger.info('getCustomerTheme with id' + id);
try {
await this.themeService.resetTheme(id);
return { theme: null };
}catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND');
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
} else {
this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message);
return { theme: null };
};
}
}
}
-12
View File
@@ -44,18 +44,6 @@ export class ThemeService implements OnModuleInit {
);
}
async resetTheme(id: string) {
const { theme } = await firstValueFrom(
this.themeService.ResetCustomerTheme({
id
}),
);
return {
theme
}
}
async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) {
if (!id) {
this.logger.error('Error - saveCustomertheme - not found id:' + id);
-25
View File
@@ -38,12 +38,6 @@ export class User {
department?: string;
@ApiProperty()
hierarchy?: string;
@ApiProperty()
bio?: string;
@ApiProperty()
companyName?: string;
@ApiProperty()
personalSite?: string;
@ApiPropertyOptional()
customer?: Customer;
@ApiProperty()
@@ -64,8 +58,6 @@ export class UserNoRolesAndCustomer extends OmitType(UserNoRoles, [
export class IUserByCustomer extends OmitType(User, ['customer']) {
@ApiPropertyOptional()
permissions?: string[];
@ApiPropertyOptional()
authProvider?: string;
}
export class CreateUserReq {
@@ -117,12 +109,6 @@ export class UpdateUserReq {
@ApiPropertyOptional()
hierarchy?: string;
@ApiPropertyOptional()
bio?: string;
@ApiPropertyOptional()
personalSite?: string;
@ApiPropertyOptional()
companyName?: string;
@ApiPropertyOptional()
roleNames?: string[];
}
@@ -188,14 +174,3 @@ export class GetAllDepartmentsRes {
@ApiProperty()
departments: string[];
}
export interface UserReporter {
name: string;
email: string;
mfaStatus: string;
status: string;
lastLogin: string;
createdAt: string;
updatedAt: string;
}
@@ -8,17 +8,6 @@ import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
+2 -36
View File
@@ -12,7 +12,6 @@ import {
Post,
Put,
Query,
Res,
UseFilters,
} from '@nestjs/common';
import {
@@ -25,13 +24,12 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -54,7 +52,6 @@ import {
UpdateUserRes,
} from './dtos/entities';
import { UsersService } from './users.service';
import { Response } from 'express';
@ApiInternalOnlyController()
@ApiTags('Users')
@@ -83,26 +80,6 @@ export class UsersController {
return await this.userService.findAllUsersByCustomerId(user.customer_id);
}
@Get('/download')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async downloadUsersInCsv(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Res() res: Response
) {
this.logger.info('downloadUsersInCsv');
this.userService.setLanguage(language);
const {
file,
filename
} = await this.userService.downloadUsersInCsv(user.customer_id);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@Get('hierarchies')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: GetAllHierarchiesRes })
@@ -266,6 +243,7 @@ export class UsersController {
}
@Patch(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: UpdateUserRes })
async updateUser(
@User() user: RequestUser,
@@ -273,18 +251,6 @@ export class UsersController {
@Param('id') id: string,
@Language() language: LanguageEnum,
) {
const isSameUser = user.user_id === id;
const isSuperAdmin = user.permissions.includes(PERMISSIONS_GROUPS.USERS.permissions.ADMIN.seqid)
if (!isSameUser && !isSuperAdmin) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
if (isSameUser && !isSuperAdmin && body.roleNames) {
// Prevent users from updating their own roles
delete body.roleNames;
}
this.logger.info('updateUser', { user });
this.userService.setLanguage(language);
return await this.userService.updateUser(body, id, user.customer_id);
-11
View File
@@ -9,17 +9,6 @@ import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
error: (...args) => args,
+7 -41
View File
@@ -21,7 +21,6 @@ import {
IUserByCustomer,
SetUserRolesReq,
UpdateUserReq,
UserReporter,
} from './dtos/entities';
import { RolesService } from '../roles/roles.service';
import { HIERARCHIES } from './hierarchies';
@@ -30,7 +29,6 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces
import { EnrichErrorCode } from 'src/utils/ErrorBuilder';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
@Injectable()
export class UsersService implements OnModuleInit {
@@ -80,34 +78,6 @@ export class UsersService implements OnModuleInit {
};
}
async downloadUsersInCsv(customerId: string) {
const { users } = await lastValueFrom(
this.usersClientService.UserFindAllByCustomerId({ customerId }),
);
const formatUsers: UserReporter[] = users.map(user => ({
createdAt: user.createdAt,
email: user.email,
lastLogin: user.lastLogin,
mfaStatus: user.mfaStatus,
name: user.name,
status: user.status,
updatedAt: user.updatedAt
}))
const parser = ParserBuilder.build<UserReporter>('csv');
const file = await parser.parse(formatUsers);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_users_${timestamp}.csv`;
return {
file,
filename
}
}
async findOneById(id: string): Promise<{ user: IUserByCustomer }> {
const { user } = await lastValueFrom(
this.usersClientService.UserFindOneById({ id }),
@@ -125,7 +95,6 @@ export class UsersService implements OnModuleInit {
return { permissions };
});
res.user.permissions = permissions;
res.user.authProvider = process.env.AUTH_PROVIDER || 'cognito';
return res;
}
@@ -149,23 +118,20 @@ export class UsersService implements OnModuleInit {
}
async updateUser(req: UpdateUserReq, id: string, customerId: string) {
const { roleNames, ...updateUserDTO } = req;
if (roleNames && roleNames.length > 0) {
const { department, hierarchy, jobTitle, name, roleNames, email } = req;
if (roleNames) {
await this.setRoles({ roleNames, userId: id }, customerId);
}
const { user } = await lastValueFrom(
this.usersClientService.UserUpdate({
department: updateUserDTO.department,
email: updateUserDTO.email,
hierarchy: updateUserDTO.hierarchy,
jobTitle: updateUserDTO.jobTitle,
name: updateUserDTO.name,
bio: updateUserDTO.bio,
companyName: updateUserDTO.companyName,
personalSite: updateUserDTO.personalSite,
name,
customerId,
id,
department,
hierarchy,
jobTitle,
email,
metabaseUserId: undefined,
}),
);
+1 -4
View File
@@ -14,10 +14,7 @@ export class ValidationPipe implements PipeTransform<any> {
return value;
}
const object = plainToInstance(metatype, value);
const errors = await validate(object, {
forbidUnknownValues: false,
whitelist: true,
});
const errors = await validate(object);
if (errors.length > 0) {
const errorMessages = errors.map((err) => err.constraints);
throw new BadRequestException(errorMessages);
-23
View File
@@ -1,23 +0,0 @@
import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
import { Cache } from 'cache-manager';
@Injectable()
export class CacheService<T> {
constructor(
@Inject(CACHE_MANAGER)
private readonly cacheManager: Cache,
) {}
async get(key: string): Promise<T | null> {
return await this.cacheManager.get<T>(key);
}
async set(key: string, value: T): Promise<void> {
await this.cacheManager.set(key, value);
}
async delete(key: string) {
await this.cacheManager.del(key);
}
}
-4
View File
@@ -1,4 +0,0 @@
export const DYNAMODB_CONFIG = {
region: () => process.env.AWS_REGION || 'us-east-1',
inputsTable: () => process.env.INPUTS_DB || 'dadosfera-inputs-prd',
};
-9
View File
@@ -1,9 +0,0 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { DynamoDBService } from './dynamodb.service';
@Module({
providers: [DynamoDBService, DadosferaLogger],
exports: [DynamoDBService],
})
export class DynamoDBModule {}

Some files were not shown because too many files have changed in this diff Show More