Compare commits

..
Author SHA1 Message Date
marcos-silva-rodrigues f3550255c4 FIX: validate if permission is public 2025-06-11 15:18:55 -03:00
87 changed files with 1027 additions and 4698 deletions
+72 -7
View File
@@ -13,6 +13,11 @@ on:
options:
- stg
- prd
push_to_dockerhub:
description: "Push image to Dockerhub?"
required: true
type: boolean
default: false
jobs:
extract_environment:
@@ -113,6 +118,23 @@ jobs:
docker compose -f build.docker-compose.yml build
docker compose -f build.docker-compose.yml push
- name: Login to Docker Hub
if: ${{inputs.push_to_dockerhub}}
uses: docker/login-action@v2
with:
username: dadosfera
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Build, Tag, and Push Image to Dockerhub
if: ${{inputs.push_to_dockerhub}}
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
run: |
docker compose -f build.docker-compose.dockerhub.yml build
docker compose -f build.docker-compose.dockerhub.yml push
# - name: Create ZIP file to Deploy AWS Beanstalk
# env:
# ENV: ${{ needs.extract_environment.outputs.environment }}
@@ -142,11 +164,54 @@ jobs:
docker system prune --volumes -a -f
docker system df
k8s-deploy:
helmfile-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: 'oracle'
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Apply
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
-137
View File
@@ -1,137 +0,0 @@
name : K8s deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
image:
description: "Image Tag"
required: true
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
oracle:
if: inputs.cloud == 'oracle'
runs-on: [self-hosted, "prd-oracle"]
env:
HOME: /home/runner
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Helm
uses: azure/setup-helm@v1
with:
version: 'v3.9.0'
- name: Install OCI CLI
env:
HOME: /home/runner
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ inputs.environment }}
IMAGE_TAG: ${{ inputs.image }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
+22 -45
View File
@@ -4,7 +4,7 @@ on:
pull_request:
branches:
- main
- beta
- stg
jobs:
extract_environment:
@@ -21,13 +21,17 @@ jobs:
fi
id: extract_environment
helmfile-check:
env:
HOME: /home/runner
helmfile-deploy:
needs: [extract_environment]
environment: ${{ needs.extract_environment.outputs.environment }}
runs-on: [self-hosted, "prd-oracle"]
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Summary
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: |
echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
- name: Checkout code
uses: actions/checkout@v3
@@ -36,28 +40,13 @@ jobs:
with:
version: 'v3.9.0'
- name: Determine DNS_HOST based on environment
id: set_dns
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
- name: Install Azure ClI
run: |
if [ "$ENV" = "prd" ]; then
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
elif [ "$ENV" = "stg" ]; then
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
fi
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Set up Python
uses: actions/setup-python@v4
@@ -68,35 +57,23 @@ jobs:
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Run Helmfile Diff
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
run: helmfile -f helmfiles/${ENV}.yaml diff
Binary file not shown.
-36
View File
@@ -1,36 +0,0 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
app: {{ .Values.app_name }}
name: {{ .Values.app_name }}
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: {{ .Values.app_name }}
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
-16
View File
@@ -1,16 +0,0 @@
maestro:
env: stg
duc_url: duc.stg.dadosfera.ai
pi_factory_url: pi-factory.stg.dadosfera.ai
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity: null
-56
View File
@@ -1,56 +0,0 @@
releases:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
- name: unimed-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
set:
- name: app_name
value: maestro-unimed
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro-unimed.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
# Customer id
- name: maestro.dedicated_proxy
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
# 10.70.0.0/16 internal network
# 137.131.167.254/32 loadbalancer
# 159.112.184.81/32 cluster ip for the uptime request ingest
- name: maestro.restricted_ip
value: "177.52.172.0/24, 189.84.160.157/32, 186.237.171.146/32, 137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
-30
View File
@@ -1,30 +0,0 @@
charts:
- name: maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
# Environment to test Network Policies
- name: private-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
set:
- name: app_name
value: maestro-private
- name: hostname
value: private-maestro.stg.dadosfera.ai
# Customer id
- name: maestro.dedicated_proxy
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
- name: replicaCount
value: 1
- name: affinity
value: null
- name: resources
value: null
- name: maestro.restricted_ip
value: "137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+516 -1241
View File
File diff suppressed because it is too large Load Diff
-10
View File
@@ -14,16 +14,6 @@ declare global {
AWS_REGION: string;
OPEN_GROUP_ID: string;
OPEN_CUSTOMER_ID: string;
DEDICATED_PROXY: string;
COOKIE_SECRET: string;
// Autodrive Configuration
AUTODRIVE_USERNAME?: string;
AUTODRIVE_PASSWORD?: string;
AUTODRIVE_BASE_URL?: string;
AUTODRIVE_MODEL?: string;
AUTODRIVE_KEY?: string;
AUTO_DRIVE_KEY?: string;
}
}
}
+22
View File
@@ -0,0 +1,22 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.dadosfera.ai
- name: hostname
value: maestro.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
- name: replicaCount
value: 2
+22
View File
@@ -0,0 +1,22 @@
charts:
- name: maestro
chart: ../maestro
values:
- ../maestro/values.yaml
set:
- name: maestro.duc_url
value: duc.stg.dadosfera.ai
- name: hostname
value: maestro.stg.dadosfera.ai
- name: maestro.pi_factory_url
value: pi-factory.dadosfera.ai
- name: maestro.in_factory_url
value: in-factory.stg.dadosfera.ai
- name: maestro.tr_factory_url
value: in-factory.dadosfera.ai
- name: maestro.open_customer_id
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
- name: maestro.open_group_id
value: e3f98a2f-7748-4981-8505-7695c8ca8218
- name: replicaCount
value: 1
+23
View File
@@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
@@ -1,16 +1,16 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.app_name }}
name: maestro
namespace: applications
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: {{ .Values.app_name }}
app: maestro
strategy:
rollingUpdate:
@@ -20,17 +20,22 @@ spec:
template:
metadata:
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
imagePullSecrets:
- name: {{ .Values.imagePullSecrets }}
nodeSelector:
"beta.kubernetes.io/os": linux
{{- if .Values.affinity }}
affinity:
{{- toYaml .Values.affinity | nindent 8 }}
{{- end }}
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: application
operator: In
values:
- general
tolerations:
- key: "kubernetes.azure.com/scalesetpriority"
@@ -43,10 +48,13 @@ spec:
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
ports:
- containerPort: {{ .Values.containerPort }}
{{- if .Values.resources }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- end }}
requests:
cpu: {{ .Values.resources.requests.cpu }}
memory: {{ .Values.resources.requests.memory }}
limits:
cpu: {{ .Values.resources.limits.cpu }}
memory: {{ .Values.resources.limits.memory }}
env:
- name: AWS_IDENTITY_POOL_ID
value: {{ .Values.maestro.aws_identity_pool_id }}
@@ -74,8 +82,6 @@ spec:
value: "logstash-pipelines.dadosfera.ai"
- name: LOGGER_GELF_PORT
value: "{{ .Values.maestro.logger_gelf_port }}"
- name: LOGGER_CONSOLE_EXTRA
value: "true"
- name: NIMBUS_BASE_URL
value: "http://nimbus-api"
- name: NPM_TOKEN
@@ -94,16 +100,6 @@ spec:
value: {{ .Values.maestro.open_customer_id }}
- name: OPEN_GROUP_ID
value: {{ .Values.maestro.open_group_id }}
- name: DEDICATED_PROXY
value: {{ .Values.maestro.dedicated_proxy }}
- name: COOKIE_SECRET
value: {{ .Values.maestro.cookie_secret }}
- name: REDIS_DATABASE
value: "{{ .Values.maestro.redis_database }}"
- name: REDIS_HOST
value: {{ .Values.maestro.redis_host }}
- name: REDIS_PORT
value: "{{ .Values.maestro.redis_port }}"
- name: JWT_PRIVATE_KEY
valueFrom:
secretKeyRef:
@@ -112,15 +108,15 @@ spec:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
name: prd-maestro
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
name: prd-maestro
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
name: prd-{{ .Values.app_name }}
name: prd-maestro
key: AWS_DEFAULT_REGION
@@ -10,12 +10,8 @@ metadata:
generation: 1
labels:
app: {{ .Values.app_name }}
{{- if .Values.maestro.dedicated_proxy}}
name: open-data-{{ .Values.app_name }}
{{- else }}
app: maestro
name: open-data
{{- end }}
namespace: applications
spec:
ingressClassName: nginx
@@ -25,7 +21,7 @@ spec:
paths:
- backend:
service:
name: {{ .Values.app_name }}
name: maestro
port:
number: {{ .Values.ingress.port }}
path: /open-data/sharing-ocean-data
+27
View File
@@ -0,0 +1,27 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
generation: 1
labels:
app: maestro
name: maestro
namespace: applications
spec:
ingressClassName: nginx
rules:
- host: {{ .Values.hostname }}
http:
paths:
- backend:
service:
name: maestro
port:
number: {{ .Values.ingress.port }}
path: /
pathType: Prefix
@@ -1,17 +1,17 @@
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: prd-{{ .Values.app_name }}
name: prd-maestro
namespace: applications
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
refreshInterval: 1h
secretStoreRef:
name: secretsmanager-prd
kind: SecretStore
target:
name: prd-{{ .Values.app_name }}
name: prd-maestro
creationPolicy: Owner
data:
- secretKey: AWS_ACCESS_KEY_ID
@@ -1,18 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.app_name }}
name: maestro
namespace: applications
labels:
app: {{ .Values.app_name }}
app: maestro
spec:
type: ClusterIP
ports:
- name: {{ .Values.app_name }}
- name: maestro
protocol: TCP
port: {{ .Values.service.port }}
targetPort: {{ .Values.service.targetPort }}
selector:
app: {{ .Values.app_name }}
app: maestro
@@ -9,7 +9,6 @@ image:
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: 1.56.0
app_name: maestro
containerPort: 3333
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
service:
@@ -43,25 +42,9 @@ maestro:
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 100
targetCPUUtilizationPercentage: 80
targetMemoryUtilizationPercentage: 80
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- product
+4 -331
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.1",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -27,11 +27,8 @@
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -42,7 +39,6 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
@@ -50,17 +46,13 @@
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
"swagger-ui-express": "^4.6.3"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
@@ -1408,9 +1400,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.10.tgz",
"integrity": "sha512-4miwovVFHtBY1VTHdW8BVmSB8m+LXfIA8uigHbeo8IwreMtTHuLpvfenT4MBSPkdVmlo9+0XBKKf+PaSqtdMAg==",
"version": "3.38.0-beta.1",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.1.tgz",
"integrity": "sha512-4+yNZMlhvEkHlxJcaeta+jtJ9owskZcg0yjKdZzBfT8PCclctgertLHmrG0+kDb3Y/na/rpPPHYULRp7AO4zDQ==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
@@ -1608,12 +1600,6 @@
"dev": true,
"license": "BSD-3-Clause"
},
"node_modules/@ioredis/commands": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
"integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==",
"license": "MIT"
},
"node_modules/@istanbuljs/load-nyc-config": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz",
@@ -2983,65 +2969,6 @@
"node": ">=18"
}
},
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/graph": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.7.tgz",
"integrity": "sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.2.0.tgz",
"integrity": "sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.1.0.tgz",
"integrity": "sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==",
"license": "MIT",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@sinonjs/commons": {
"version": "1.8.6",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz",
@@ -3727,13 +3654,6 @@
"@types/node": "*"
}
},
"node_modules/@types/cache-manager": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/@types/cache-manager/-/cache-manager-4.0.6.tgz",
"integrity": "sha512-8qL93MF05/xrzFm/LSPtzNEOE1eQF3VwGHAcQEylgp5hDSTe41jtFwbSYAPfyYcVa28y1vYSjIt0c1fLLUiC/Q==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/connect": {
"version": "3.4.38",
"resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz",
@@ -3744,16 +3664,6 @@
"@types/node": "*"
}
},
"node_modules/@types/cookie-parser": {
"version": "1.4.9",
"resolved": "https://registry.npmjs.org/@types/cookie-parser/-/cookie-parser-1.4.9.tgz",
"integrity": "sha512-tGZiZ2Gtc4m3wIdLkZ8mkj1T6CEHb35+VApbL2T14Dew8HA7c+04dmKqsKRNC+8RJPm16JEK0tFSwdZqubfc4g==",
"dev": true,
"license": "MIT",
"peerDependencies": {
"@types/express": "*"
}
},
"node_modules/@types/cookiejar": {
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz",
@@ -3813,16 +3723,6 @@
"@types/send": "*"
}
},
"node_modules/@types/express-session": {
"version": "1.18.1",
"resolved": "https://registry.npmjs.org/@types/express-session/-/express-session-1.18.1.tgz",
"integrity": "sha512-S6TkD/lljxDlQ2u/4A70luD8/ZxZcrU5pQwI1rVXCiaVIywoFgbA+PIUNDjPhQpPdK0dGleLtYc/y7XWBfclBg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/express": "*"
}
},
"node_modules/@types/graceful-fs": {
"version": "4.1.9",
"resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz",
@@ -5337,41 +5237,6 @@
"node": ">= 0.8"
}
},
"node_modules/cache-manager": {
"version": "5.7.6",
"resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
"integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
"license": "MIT",
"dependencies": {
"eventemitter3": "^5.0.1",
"lodash.clonedeep": "^4.5.0",
"lru-cache": "^10.2.2",
"promise-coalesce": "^1.1.2"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/cache-manager-ioredis-yet": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cache-manager-ioredis-yet/-/cache-manager-ioredis-yet-1.2.2.tgz",
"integrity": "sha512-o03N/tQxfFONZ1XLGgIxOFHuQQpjpRdnSAL1THG1YWZIVp1JMUfjU3ElSAjFN1LjbJXa55IpC8waG+VEoLUCUw==",
"deprecated": "With cache-manager v6 we now are using Keyv",
"license": "MIT",
"dependencies": {
"cache-manager": "^5.2.3",
"ioredis": "^5.3.2"
},
"engines": {
"node": ">= 16.17.0"
}
},
"node_modules/cache-manager/node_modules/lru-cache": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
"license": "ISC"
},
"node_modules/call-bind": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz",
@@ -5649,15 +5514,6 @@
"node": ">=0.8"
}
},
"node_modules/cluster-key-slot": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
@@ -5884,28 +5740,6 @@
"node": ">= 0.6"
}
},
"node_modules/cookie-parser": {
"version": "1.4.7",
"resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.7.tgz",
"integrity": "sha512-nGUvgXnotP3BsjiLX2ypbQnWoGUPIIfHQNZkkC668ntrzGWEZVW70HDEB1qnNGMicPje6EttlIgzo51YSwNQGw==",
"license": "MIT",
"dependencies": {
"cookie": "0.7.2",
"cookie-signature": "1.0.6"
},
"engines": {
"node": ">= 0.8.0"
}
},
"node_modules/cookie-parser/node_modules/cookie": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
@@ -6193,15 +6027,6 @@
"node": ">=0.4.0"
}
},
"node_modules/denque": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz",
"integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==",
"license": "Apache-2.0",
"engines": {
"node": ">=0.10"
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -6977,12 +6802,6 @@
"node": ">= 0.6"
}
},
"node_modules/eventemitter3": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
"integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==",
"license": "MIT"
},
"node_modules/events": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
@@ -7651,15 +7470,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"license": "MIT",
"engines": {
"node": ">= 4"
}
},
"node_modules/gensync": {
"version": "1.0.0-beta.2",
"resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
@@ -8235,30 +8045,6 @@
"node": ">= 0.10"
}
},
"node_modules/ioredis": {
"version": "5.6.1",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
"integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
"license": "MIT",
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
"lodash.isarguments": "^3.1.0",
"redis-errors": "^1.2.0",
"redis-parser": "^3.0.0",
"standard-as-callback": "^2.1.0"
},
"engines": {
"node": ">=12.22.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/ioredis"
}
},
"node_modules/ip-address": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz",
@@ -9258,15 +9044,6 @@
"url": "https://github.com/chalk/supports-color?sponsor=1"
}
},
"node_modules/jose": {
"version": "4.15.9",
"resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz",
"integrity": "sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -9589,30 +9366,12 @@
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
"license": "MIT"
},
"node_modules/lodash.clonedeep": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
"integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==",
"license": "MIT"
},
"node_modules/lodash.defaults": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
"integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==",
"license": "MIT"
},
"node_modules/lodash.includes": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
"integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==",
"license": "MIT"
},
"node_modules/lodash.isarguments": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
"integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==",
"license": "MIT"
},
"node_modules/lodash.isboolean": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz",
@@ -10264,15 +10023,6 @@
"integrity": "sha512-CsubGNxhIEChNY4cXYuA6KXafztzHqzLLZ/y3Kasf3A+sa3lL9thq3z+7o0pZqzEinjXT6lXDPAfVWI59dUyzQ==",
"license": "MIT"
},
"node_modules/object-hash": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz",
"integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==",
"license": "MIT",
"engines": {
"node": ">= 6"
}
},
"node_modules/object-identity-map": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/object-identity-map/-/object-identity-map-1.0.2.tgz",
@@ -10318,15 +10068,6 @@
"node": ">= 0.4"
}
},
"node_modules/oidc-token-hash": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.1.0.tgz",
"integrity": "sha512-y0W+X7Ppo7oZX6eovsRkuzcSM40Bicg2JEJkDJ4irIt1wsYAP5MLSNv+QAogO8xivMffw/9OvV3um1pxXgt1uA==",
"license": "MIT",
"engines": {
"node": "^10.13.0 || >=12.0.0"
}
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@@ -10372,21 +10113,6 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/openid-client": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.7.1.tgz",
"integrity": "sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==",
"license": "MIT",
"dependencies": {
"jose": "^4.15.9",
"lru-cache": "^6.0.0",
"object-hash": "^2.2.0",
"oidc-token-hash": "^5.0.3"
},
"funding": {
"url": "https://github.com/sponsors/panva"
}
},
"node_modules/optional-js": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/optional-js/-/optional-js-2.3.0.tgz",
@@ -11089,15 +10815,6 @@
"node": ">=0.4.0"
}
},
"node_modules/promise-coalesce": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/promise-coalesce/-/promise-coalesce-1.1.2.tgz",
"integrity": "sha512-zLaJ9b8hnC564fnJH6NFSOGZYYdzrAJn2JUUIwzoQb32fG2QAakpDNM+CZo1km6keXkRXRM+hml1BFAPVnPkxg==",
"license": "BSD-3-Clause",
"engines": {
"node": ">=16"
}
},
"node_modules/prompts": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz",
@@ -11477,44 +11194,6 @@
"node": ">= 0.10"
}
},
"node_modules/redis": {
"version": "4.7.1",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.7.1.tgz",
"integrity": "sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==",
"license": "MIT",
"workspaces": [
"./packages/*"
],
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.6.1",
"@redis/graph": "1.1.1",
"@redis/json": "1.0.7",
"@redis/search": "1.2.0",
"@redis/time-series": "1.1.0"
}
},
"node_modules/redis-errors": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz",
"integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==",
"license": "MIT",
"engines": {
"node": ">=4"
}
},
"node_modules/redis-parser": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz",
"integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==",
"license": "MIT",
"dependencies": {
"redis-errors": "^1.0.0"
},
"engines": {
"node": ">=4"
}
},
"node_modules/reflect-metadata": {
"version": "0.1.14",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz",
@@ -12202,12 +11881,6 @@
"integrity": "sha512-oeVtt7eWQS+Na6F//S4kJ2K2VbRlS9D43mAlMyVpVWovy9o+jfgH8O9agzANzaiLjclA0oYzUXEM4PurhSUChw==",
"license": "MIT"
},
"node_modules/standard-as-callback": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz",
"integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==",
"license": "MIT"
},
"node_modules/statuses": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz",
+1 -9
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.1",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -45,11 +45,8 @@
"@nestjs/swagger": "^6.3.0",
"@nestjs/testing": "^9.4.3",
"axios": "^0.27.2",
"cache-manager": "^5.1.4",
"cache-manager-ioredis-yet": "^1.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.7",
"cron-parser": "^4.9.0",
"csv": "^6.3.11",
"dotenv": "^14.3.2",
@@ -60,7 +57,6 @@
"jwk-to-pem": "^2.0.5",
"mixpanel": "^0.17.0",
"ms": "^3.0.0-canary.1",
"openid-client": "^5.7.1",
"passport": "^0.6.0",
"passport-facebook": "^3.0.0",
"passport-forcedotcom": "^0.2.0",
@@ -68,7 +64,6 @@
"passport-hubspot-oauth2": "^1.0.3",
"passport-mailchimp": "^1.1.0",
"puppeteer": "^24.7.2",
"redis": "^4.5.1",
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.5.5",
@@ -78,10 +73,7 @@
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
"@types/jsonwebtoken": "^8.5.9",
"@types/jwk-to-pem": "^2.0.1",
+2 -10
View File
@@ -28,10 +28,7 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
import { OpenDataModule } from './modules/open-data/open-data.module';
import { ThemeModule } from './modules/theme/theme.module';
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
import { AssignModule } from './modules/assign/assign.module';
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
import { ApiKeyModule } from './modules/api-key/api-key.module';
@Module({
@@ -66,15 +63,10 @@ import { ApiKeyModule } from './modules/api-key/api-key.module';
CustomersModule,
OpenDataModule,
ThemeModule,
NetworkPolicyModule,
AssignModule,
ShareMetadataModule,
NetworkPolicyModule,
ApiKeyModule,
IdentityProviderModule,
NetworkPolicyModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
NetworkPolicyModule,
ApiKeyModule
],
})
export class AppModule {}
+12 -20
View File
@@ -17,7 +17,6 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -100,7 +99,6 @@ describe('authentication.guard', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: []
};
beforeAll(async () => {
@@ -122,12 +120,6 @@ describe('authentication.guard', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [NoClassAuthController, ClassAuthConditionController],
}).compile();
@@ -448,18 +440,18 @@ describe('authentication.guard', () => {
NoClassAuthTest(null, null);
ClassAuthConditionTest(null, null);
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
// NoClassAuthTest(tokenZ, ['zendesk']);
// ClassAuthConditionTest(tokenZ, ['zendesk']);
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
NoClassAuthTest(tokenZ, ['zendesk']);
ClassAuthConditionTest(tokenZ, ['zendesk']);
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
// NoClassAuthTest(tokenM, ['metabase']);
// ClassAuthConditionTest(tokenM, ['metabase']);
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
NoClassAuthTest(tokenM, ['metabase']);
ClassAuthConditionTest(tokenM, ['metabase']);
// const tokenZM = CreateToken([
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
// ]);
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
const tokenZM = CreateToken([
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
]);
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
});
@@ -4,7 +4,6 @@ import {
OnApplicationBootstrap,
ExecutionContext,
Inject,
ForbiddenException,
} from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import assert from 'assert';
@@ -136,18 +135,6 @@ export class AuthenticationGuard
return false;
}
// Bloquear outros customer de usar o maestor dedicado
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
}
// Bloquear o customer de acesso o maestro publico
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
}
request.accessTokenPayload = accessTokenPayload;
request.user = {
user_id: accessTokenPayload.user_id,
+27 -22
View File
@@ -340,6 +340,16 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
},
},
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
@@ -352,25 +362,6 @@ export const PERMISSIONS_GROUPS = {
},
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
'en-us': 'Analyze | Embedding',
'es-es': 'Analizar | Incorporación',
},
permissions: {
EMBED_ANALYTICS: {
seqid: 44,
claim: 'catalog:embed',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
'en-us': 'Access Embedding analytics Module',
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
}
},
CONNECTORS: {
title: {
'pt-br': 'Conectores',
@@ -612,6 +603,22 @@ export const PERMISSIONS_GROUPS = {
},
},
};
export const PUBLIC_PERMISSIONS_SEQID = [
46, // AUTH.GENERATE_TOKEN
23, 13, 29, 5, // PIPELINE
37, 38, // CONNECTION
35, // NETWORK_CONFIG
7, 19, 25, 1, // CATALOG
14, // SNOWFLAKE
11, // DATAVIZ
31, // INTELLIGENCE
34, // USERS
43, // PROCESS
47 // CUSTOMER
];
export interface DadosferaModule {
name: string;
description: string;
@@ -623,9 +630,7 @@ export const DADOSFERA_MODULES_KEYS = {
LOG_DASHBOARD: 'logs-dashboard',
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii',
EMBED: 'embedded-analytics',
EMBED_ASSIGNED: 'embed-assigned',
PII: 'pii'
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+1 -9
View File
@@ -9,7 +9,6 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
const logger = {
info: (...args) => args,
@@ -53,7 +52,6 @@ describe('user.decorator', () => {
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
customer_name: 'dadosfera',
customer_tier: 'BASIC',
customer_modules: [],
access_token: '',
};
@@ -76,12 +74,6 @@ describe('user.decorator', () => {
provide: APP_GUARD,
useClass: AuthenticationGuard,
},
{
provide: ApiKeyService,
useValue: {
get: () => Promise.resolve(null)
}
}
],
controllers: [UserController],
}).compile();
@@ -183,5 +175,5 @@ describe('user.decorator', () => {
const token = CreateToken();
fakeUserPayload.access_token = token;
// UserTest(token);
UserTest(token);
});
-4
View File
@@ -9,8 +9,6 @@ import { AppModule } from './app.module';
import { writeFileSync } from 'fs';
import { execSync } from 'child_process';
import { INestApplication } from '@nestjs/common';
import cookieParser from 'cookie-parser';
async function bootstrap() {
DadosferaLogger.setupLogger({
serviceName: 'maestro',
@@ -25,11 +23,9 @@ async function bootstrap() {
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
+5 -1
View File
@@ -1,12 +1,13 @@
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
import { ApiKeyService } from './api-key.service';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { Authenticated, RequireAllPermissions } from 'src/decorators/authentication.decorator';
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('api-key')
@Authenticated()
@@ -26,6 +27,7 @@ export class ApiKeyController {
@Post()
@ApiResponse({ type: CreateApiKeyResponseDto })
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
this.logger.info('POST /api-key', {
permissions: createApiKeyDto.permissions,
@@ -41,6 +43,7 @@ export class ApiKeyController {
@Get()
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
this.logger.info('GET /api-key', {
method: 'findAll'
@@ -54,6 +57,7 @@ export class ApiKeyController {
}
@Delete(':id')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
this.logger.info('DELETE /api-key/:id', {
id,
+46 -11
View File
@@ -1,38 +1,73 @@
import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
import {
Injectable,
Inject,
OnModuleInit,
BadRequestException,
} from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
import {
CreateApiKeyDto,
CreateApiKeyResponseDto,
ApiKeyBaseResponseDto,
} from './dto/api-key.dto';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { PackTheMetadata } from '../../utils/ PackTheMetadata';
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PUBLIC_PERMISSIONS_SEQID } from 'src/authentication/permissions.enum';
@Injectable()
export class ApiKeyService implements OnModuleInit {
private apiKeyService: ApiKeyWriteProtoService;
constructor(
@Inject(DucClient.name) private readonly client: ClientGrpc,
@Inject(DucClient.name) private readonly client: ClientGrpc
) {}
onModuleInit() {
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(
ProtoServices.ApiKeyWriteProtoService,
);
}
create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
create(
createApiKeyDto: CreateApiKeyDto,
user: RequestUser,
): Promise<CreateApiKeyResponseDto> {
const metadata = PackTheMetadata(user);
return lastValueFrom(this.apiKeyService.CreateApiKey({
permissions: createApiKeyDto.permissions
}, metadata));
const invalidPermissions = [];
for (const permission of createApiKeyDto.permissions) {
if (!PUBLIC_PERMISSIONS_SEQID.includes(permission)) {
invalidPermissions.push(permission);
}
}
if (invalidPermissions.length > 0) {
throw new BadRequestException(
`Invalid permissions: ${invalidPermissions.join(', ')}`,
);
}
return lastValueFrom(
this.apiKeyService.CreateApiKey(
{
permissions: createApiKeyDto.permissions,
},
metadata,
),
);
}
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
const metadata = PackTheMetadata(user);
console.log(metadata)
console.log(metadata);
const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
const data = await lastValueFrom(
this.apiKeyService.ListApiKeys({}, metadata),
);
return data.api_keys;
}
-33
View File
@@ -1,33 +0,0 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Controller('assign')
@Authenticated()
export class AssignController {
constructor(private readonly assignService: AssignService) {}
@Put('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
}
@Get('/public-key')
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
}
}
-15
View File
@@ -1,15 +0,0 @@
import { Module } from '@nestjs/common';
import { AssignService } from './assign.service';
import { AssignController } from './assign.controller';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [AssignController],
providers: [AssignService, DadosferaLogger]
})
export class AssignModule {}
-38
View File
@@ -1,38 +0,0 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Metadata } from '@grpc/grpc-js';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from 'src/modules/duc/client.config';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
@Injectable()
export class AssignService implements OnModuleInit {
ducService: AssingProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<AssingProtoService>(
ProtoServices.AssingProtoService,
);
}
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
return data;
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -1,3 +0,0 @@
export class CreateAssignDto {
publicKey: string;
}
+14 -197
View File
@@ -12,7 +12,6 @@ import {
Redirect,
Req,
Param,
Res,
} from '@nestjs/common';
import {
ApiHeaders,
@@ -27,7 +26,7 @@ import {
AuthConfirmResetPasswordRequest,
AuthEnableTotpMfaRequest,
AuthDisableTotpMfaRequest,
AuthVerifyTotpMfaRequest
AuthVerifyTotpMfaRequest,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@@ -46,21 +45,14 @@ import {
AuthSignInRes,
BulkEditRequest,
} from './dtos/login';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { AuthGuard } from '@nestjs/passport';
import { Request, Response } from 'express';
import { Request } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt, { JwtPayload } from 'jsonwebtoken';
import jwt from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
refreshToken?: string;
userId?: string
}
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@@ -94,66 +86,13 @@ export class AuthController {
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
const data = await this.authClient.signIn({ username, password, totp }, metadata);
if (data.tokens) {
this.addTokenInCookie(res, {
accessToken: data.tokens.accessToken,
refreshToken: data.tokens.refreshToken,
userId: data.user.id
});
}
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
throw error;
}
}
@Post('sign-out')
@HttpCode(HttpStatus.NO_CONTENT)
async signOut(
@Language() language: LanguageEnum,
@Res() res: Response,
) {
try {
this.logger.info('/auth - SignOut');
const exp = 1000 * 60 * 3;
res.cookie('ddf-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
res.cookie('ddf-refresh-auth', '', {
domain: 'dadosfera.local',
maxAge: Date.now() - exp,
expires: new Date(),
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
this.logger.info('Clean cookie sessions');
return res.send();
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
}
@Headers('origin') origin = '',
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const frontHost = origin.replace(/^https?:\/\//, '');
const metadata = PackTheMetadata({ language, custom_host: frontHost });
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
return this.authClient.signIn({ username, password, totp }, metadata);
}
@Post('refresh-access-token')
@@ -163,25 +102,17 @@ export class AuthController {
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
@Headers('origin') origin: string,
@Res() res: Response,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, customerName: customer_name } = body;
const frontHost = origin.replace(/^https?:\/\//, '');
const { refreshToken, userId } = body;
const metadata = PackTheMetadata({
language,
customer_name,
custom_host: frontHost,
});
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
return res.send(data);
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
}
@ApiInternalOnlyEndpoint()
@@ -489,118 +420,4 @@ export class AuthController {
return this.authClient.resetUsers(body.users, metadata);
}
@Get('me')
async getMe(@Req() req: Request, @Res() res: Response) {
this.logger.info('GET /auth/me ')
// Lê cookies
const accessToken = req.cookies['ddf-auth'];
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
this.logger.info('Token is invalid')
this.logger.info('Has Refresh Token: '+ Boolean(refreshToken))
// Se access token inválido, tenta refresh
if (!refreshToken || !userId) {
this.logger.error('Invalid refresh token or customer name');
return res.status(401).json({ error: 'Not authenticated' });
}
try {
// Chama refreshAccessToken
const metadata = PackTheMetadata({
});
this.logger.info('Call Refresh Token')
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
this.logger.info('Finish Refresh Token')
// Retorna novo access token e dados mínimos
this.addTokenInCookie(res, {
accessToken: data.accessToken,
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
}
}
}
private addTokenInCookie(res: Response, data: CookiesValues) {
let exp = 1000 * 60 * 5; // 5 minutes
if (data.accessToken) {
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: 'stg.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
sameSite: 'none', // Necessário para cookies em requisições cross-site
});
}
}
}
+7 -51
View File
@@ -1,10 +1,10 @@
import { OnModuleInit, Inject, Injectable, ForbiddenException } from '@nestjs/common';
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
import { ClientGrpc } from '@nestjs/microservices';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -17,22 +17,16 @@ import {
AuthResetPasswordRequest,
AuthVerifyResetPasswordCodeRequest,
AuthConfirmResetPasswordRequest,
AuthSignInResponse,
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -45,10 +39,6 @@ export class AuthClientService implements OnModuleInit {
this.authService = this.grpcClient.getService<AuthServiceInterface>(
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async getPublicKeys() {
@@ -63,59 +53,25 @@ export class AuthClientService implements OnModuleInit {
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
}
checkDedicatedProxy({
customer
}: AuthSignInResponse) {
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
this.logger.info('SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY);
this.logger.info('Customer ID: ' + customer.id);
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
throw new ForbiddenException();
}
// Bloquear o customer de acesso o maestro publico
this.logger.info('Check if customer have network policy: ' + customer.modules);
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
throw new ForbiddenException();
}
}
async signIn(
{ username, password, totp }: AuthSignInRequest,
metadata: Metadata,
) {
this.logger.info('SignIn');
let result: AuthSignInResponse;
try {
result = await lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
} catch (error) {
this.logger.error('SignIn - Error during sign-in');
this.logger.error(error);
throw error;
}
if (result.customer) {
this.checkDedicatedProxy(result);
}
return result
return lastValueFrom(
this.authService.AuthSignIn({ username, password, totp }, metadata),
);
}
async refreshAccessToken(
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
{ refreshToken }: AuthRefreshAccessTokenRequest,
metadata: Metadata,
) {
this.logger.info('RefreshAccessToken');
return lastValueFrom(
this.authService.AuthRefreshAccessToken({ refreshToken, userId }, metadata),
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
);
}
+1 -1
View File
@@ -122,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
refreshToken: string;
@ApiProperty()
userId: string;
customerName: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
@@ -26,7 +26,6 @@ export class GoogleLoginStrategy extends PassportStrategy(
callbackURL: oauthSecrets['google-login'].redirect_uri,
scope: ['email', 'profile', 'openid'],
};
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
const verify = (
accessToken: string,
refreshToken: string,
+48 -153
View File
@@ -22,9 +22,6 @@ import {
ApiHeaders,
ApiOkResponse,
ApiTags,
ApiOperation,
ApiParam,
ApiResponse,
} from '@nestjs/swagger';
import {
Authenticated,
@@ -35,7 +32,7 @@ import {
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
BatchRemoveRlsRulesRequest,
@@ -91,6 +88,10 @@ export class CatalogController {
@Query() query: ICatalogAllRequest,
): Promise<ICatalogAllResponse> {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog - searchCatalog`, {
user_id,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
@@ -116,53 +117,15 @@ export class CatalogController {
return res;
}
@Get('/download')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async dowloadAsserts(
@User() user: RequestUser,
@Query() query: ICatalogAllRequest,
@Res() res: Response
) {
const { user_id, customer_name, customer_id, username, permissions } = user;
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const roles = await this.catalogService.getUserRolesIds(user_id);
const metadata = PackTheMetadata({
user_id,
customer_id,
customer_name,
username,
roles,
is_data_manager,
});
const {
file,
filename
} = await this.catalogService.downloadAssets(
query,
metadata,
customer_id,
);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user;
const { pipeline, object } = query;
this.logger.info(`/catalog - ON GET DATA ASSET BY PIPELINE AND OBJECT`, {
username,
customer_name,
});
if (!pipeline || !object) {
throw new BadRequestException('Query params not provided');
@@ -215,6 +178,10 @@ export class CatalogController {
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
)
async findAllTags(@Body() body) {
this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const { user_id, customer, customer_id } = body.info;
const metadata = PackTheMetadata({
@@ -240,6 +207,10 @@ export class CatalogController {
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`GET /data-asset/${id}`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
@@ -292,6 +263,10 @@ export class CatalogController {
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
this.logger.info(`/catalog - ON GET ONE DASHBOARD METABASE ROUTE`, {
username,
customer_name,
});
const is_data_manager = permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
@@ -343,6 +318,10 @@ export class CatalogController {
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - columns-metadata`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
@@ -370,6 +349,10 @@ export class CatalogController {
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username, customer_modules } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
@@ -395,31 +378,24 @@ export class CatalogController {
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IDocsResponse> {
try {
const { customer_name, customer_id, user_id, username } = user;
const { customer_name, customer_id, user_id, username } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
id,
});
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const metadata = PackTheMetadata({
customer_name,
customer_id,
user_id,
username,
language,
});
const docs = await this.catalogService.getDataDocs(id, metadata);
const docs = await this.catalogService.getDataDocs(id, metadata);
return { docs };
} catch (error) {
this.logger.error(`Error in getDataAssetDocs for id ${id}: ${error.message}`);
this.logger.error(`Error details: ${JSON.stringify(error)}`);
throw error;
}
return { docs };
}
@Put('data-asset/:id')
@@ -468,6 +444,10 @@ export class CatalogController {
) {
const { user_id, customer_name } = user;
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
user_id,
customer_name,
});
const res = await this.catalogService.createDataDocs({
table_id,
@@ -480,7 +460,6 @@ export class CatalogController {
return res;
}
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions')
async manageDataAssetPermissions(
@@ -906,88 +885,4 @@ export class CatalogController {
this.logger.error(error.message);
}
}
@Post('/data-asset/:nimbus_id/docs/ai')
@ApiOperation({
summary: 'Save documentation for data asset',
description: 'Saves documentation content for a data asset',
})
@ApiParam({
name: 'nimbus_id',
description: 'Nimbus ID of the data asset',
type: 'string',
})
@ApiResponse({
status: 201,
description: 'Documentation saved successfully',
})
async saveDocumentation(
@Param('nimbus_id') nimbusId: string,
@Body() body: { docs: string },
@User() user: RequestUser,
) {
const metadata = PackTheMetadata(user);
try {
await this.catalogService.updateDataAssetDocumentation(nimbusId, body.docs, metadata);
return {
message: 'Documentation saved successfully',
};
} catch (error) {
this.logger.error(`Error saving documentation for ${nimbusId}: ${error.message}`);
throw error;
}
}
@Post('/data-asset/:nimbus_id/docs/generate-ai')
@ApiOperation({
summary: 'Generate AI documentation for data asset',
description: 'Generates comprehensive documentation for a data asset using AI (Autodrive)',
})
@ApiParam({
name: 'nimbus_id',
description: 'Nimbus ID of the data asset',
type: 'string',
})
@ApiResponse({
status: 201,
description: 'AI documentation generated successfully',
schema: {
type: 'object',
properties: {
message: { type: 'string' },
documentation: { type: 'string' },
},
},
})
@ApiResponse({
status: 400,
description: 'Bad request - invalid nimbus_id or missing data',
})
@ApiResponse({
status: 404,
description: 'Data asset not found',
})
@ApiResponse({
status: 500,
description: 'Internal server error during AI generation',
})
async generateAiDocumentation(
@Param('nimbus_id') dataAssetId: string,
@User() user: RequestUser,
) {
const metadata = PackTheMetadata(user);
try {
const result = await this.catalogService.generateAiDocumentation(dataAssetId, metadata, user);
return {
message: 'AI documentation generated successfully',
documentation: result,
};
} catch (error) {
this.logger.error(`Error generating AI documentation for ${dataAssetId}: ${error.message}`);
throw error;
}
}
}
}
+1 -6
View File
@@ -3,15 +3,12 @@ import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { CatalogController } from './catalog.controller';
import { CatalogService } from './catalog.service';
import { CatalogClientConfiguration } from './catalog-client';
import { ClientsModule } from '@nestjs/microservices';
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
import { UsersModule } from '../users/users.module';
import { RolesModule } from '../roles/roles.module';
import { CustomersModule } from '../customers/customers.module';
import { ShareModule } from './share/share.module';
import { CatalogService } from './catalog.service';
import { MixpanelModule } from '../mixpanel/mixpanel.module';
const client = new CatalogClientConfiguration();
@@ -21,8 +18,6 @@ const client = new CatalogClientConfiguration();
OldPipelineModule,
UsersModule,
RolesModule,
CustomersModule,
ShareModule,
],
controllers: [CatalogController],
providers: [CatalogService, DadosferaLogger],
+9 -611
View File
@@ -1,5 +1,4 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import FormData from 'form-data';
import {
WriteService,
ReadService,
@@ -26,7 +25,6 @@ import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import {
AssetReporter,
BatchRemoveRlsRulesRequest,
IUpdateDataRequest,
TriggerCatalogReq,
@@ -37,22 +35,14 @@ import {
GetRlsRulesRequest,
PiiMetadata,
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
import { writeFileSync } from 'fs';
import path from 'path';
import { HtmlParser } from 'src/utils/FileParser/html-parser';
import { PiiDto } from './dtos/pii.dto';
import { CsvParser } from 'src/utils/FileParser/csv-parser';
import { PDFParser } from 'src/utils/FileParser/pdf-parser';
import { TypeParser } from 'src/utils/FileParser/parser-types';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
import { AI_DOCUMENTATION_PROMPT, AI_DOCUMENTATION_CONFIG } from './prompts/ai-documentation.prompt';
import { AUTODRIVE_CONSTANTS, GEOGRAPHIC_KEYS, COMMON_COUNTRIES } from './constants/autodrive.constants';
import {
AutodriveCredentials,
AutodriveAskPayload,
AutodriveAskResponse,
AutodriveAnswerResponse,
AutodriveUploadResponse,
DatasetStatusResponse,
ColumnData,
ColumnsMetadata,
DataPreview,
FormattedDataForAI
} from './types/ai-documentation.types';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -250,34 +240,6 @@ class CatalogService implements OnModuleInit {
return { data_assets: response, total };
}
async downloadAssets(
query: Record<string, any>,
metadata: Metadata,
customer_id: string,
) {
const data = await this.searchDataAssets(query, metadata, customer_id);
const formatData = data.data_assets.map(asset => ({
id: asset.id,
display_name: asset.display_name,
data_asset_type: asset.data_asset_type,
created_at: asset.created_at,
tags: '[' + asset.tags.join(', ') + ']'
}))
const parser = ParserBuilder.build<AssetReporter>('csv');
const file = await parser.parse(formatData);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_assets_${timestamp}.csv`;
return {
file,
filename
}
}
async getOneDataAsset(data: {
id: string;
customer_id: string;
@@ -351,44 +313,15 @@ class CatalogService implements OnModuleInit {
}
async getDataDocs(id: string, metadata: Metadata) {
let documentation: string;
try {
const response = await lastValueFrom(
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
const doc = response.documentation;
documentation = doc;
if (!documentation) {
this.logger.warn(`No documentation found for id: ${id}`);
return null;
}
// Verificar se a documentação é um JSON válido
if (documentation.trim().startsWith('{') || documentation.trim().startsWith('[')) {
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
} else {
this.logger.warn(`Documentation is not JSON format, returning as raw text for id: ${id}`);
return { raw_documentation: documentation };
}
} catch (error) {
this.logger.error(`getDataDocs failed for id ${id}: ${error.message}`);
// Se for erro de JSON parsing, tentar retornar a documentação como string
if (error.message.includes('JSON') || error.message.includes('parse')) {
this.logger.warn(`JSON parsing failed, returning raw documentation for id: ${id}`);
return { raw_documentation: documentation || 'No documentation available' };
}
throw error;
}
}
async getDatasetPreview(id: string, metadata: Metadata) {
try {
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id, type: undefined },
@@ -397,35 +330,19 @@ class CatalogService implements OnModuleInit {
);
const result = JSON.parse(preview);
return result;
} catch (error) {
this.logger.error(`getDatasetPreview failed for id ${id}: ${error.message}`);
throw error;
}
}
async getDatasetColumnsMetadata(id: string, metadata: Metadata) {
try {
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id, type: undefined },
metadata,
),
);
if (!columns_metadata || columns_metadata.length <= 2) {
this.logger.warn(`Empty or minimal response from gRPC: "${columns_metadata}"`);
throw new Error('Empty response from gRPC service');
}
const result = JSON.parse(columns_metadata);
return result;
} catch (error) {
this.logger.error(`getDatasetColumnsMetadata failed for id ${id}: ${error.message}`);
throw error;
}
}
async createDataDocs(body) {
const nimbusUrl = this._getNimbusUrl(body);
const { data } = await axios.post(
@@ -435,7 +352,6 @@ class CatalogService implements OnModuleInit {
return data;
}
async findAllTags(data, metadata) {
this.logger.info('CatalogService - findAllCustomerTags');
@@ -666,524 +582,6 @@ class CatalogService implements OnModuleInit {
);
return res;
}
private capitalizeFirst(str: string): string {
return str.charAt(0).toUpperCase() + str.slice(1);
}
async updateDataAssetDocumentation(nimbusId: string, documentation: string, metadata: Metadata): Promise<void> {
try {
if (!documentation) {
this.logger.warn(`Documentation is undefined for ${nimbusId}`);
return;
}
const changes = {
documentation: documentation
};
await lastValueFrom(
this.catalogWriteService.UpdateDataAsset(
{ id: nimbusId, changes: JSON.stringify(changes) },
metadata,
),
);
this.logger.info(`Documentation updated successfully for ${nimbusId}`);
} catch (error) {
this.logger.error(`Error updating documentation for ${nimbusId}: ${error.message}`);
throw error;
}
}
async generateAiDocumentation(dataAssetId: string, metadata: Metadata, user?: any): Promise<string> {
this.logger.info(`Generating AI documentation for data asset: ${dataAssetId}`);
try {
const customer_id = metadata.get('customer_id')?.[0]?.toString();
// 1. Buscar metadados do data asset
const dataAsset = await this.getOneDataAsset({
id: dataAssetId,
customer_id,
metadata
});
if (!dataAsset || !dataAsset.data_asset || !dataAsset.data_asset.nimbus_id) {
throw new Error("Nimbus ID not found in data asset metadata");
}
const nimbusId = dataAsset.data_asset.nimbus_id.toString();
// 2. Buscar informações das colunas
let columnsData = null;
try {
columnsData = await this.getDatasetColumnsMetadata(dataAssetId, metadata);
} catch (error) {
this.logger.warn(`Failed to get column metadata: ${error.message}`);
columnsData = null;
}
// 2.1. Fallback para dados das colunas
const dataAssetWithColumns = dataAsset as any;
if (!columnsData && dataAssetWithColumns.columns && Array.isArray(dataAssetWithColumns.columns)) {
columnsData = { columns: dataAssetWithColumns.columns };
} else if (!columnsData) {
const numColumns = parseInt(dataAsset.data_asset.num_columns) || 0;
if (numColumns > 0) {
columnsData = {
columns: Array.from({ length: numColumns }, (_, i) => ({
name: `COLUMN_${i + 1}`,
type: 'UNKNOWN',
description: 'Column information not available',
nullable: 'UNKNOWN'
}))
};
}
}
// 3. Buscar preview dos dados (opcional)
let dataPreview = [];
try {
dataPreview = await this.getDatasetPreview(nimbusId, metadata);
} catch (error) {
this.logger.warn(`Failed to get data preview: ${error.message}`);
dataPreview = [];
}
// 4. Formatar dados e chamar Autodrive
const combinedText = this.formatDataForAI(dataAsset, dataPreview, columnsData);
const documentation = await this.callAutodriveForDocumentation(combinedText, user?.access_token);
// 5. Salvar a documentação
const createDataDocsPayload = {
table_id: dataAsset.data_asset.nimbus_id,
docs: documentation,
info: {
customer: user.customer_name,
},
};
await this.createDataDocs(createDataDocsPayload);
this.logger.info(`AI documentation generated and saved successfully for ${dataAssetId}, length: ${documentation.length} chars`);
return documentation;
} catch (error) {
this.logger.error(`Error generating AI documentation for ${dataAssetId}: ${error.message}`);
let errorMessage = error.message;
if (error.response) {
errorMessage = `API Error ${error.response.status}: ${error.response.data?.detail || error.message}`;
}
throw new HttpException(
`Failed to generate AI documentation: ${errorMessage}`,
HttpStatus.INTERNAL_SERVER_ERROR,
);
}
}
private formatDataForAI(dataAsset: any, dataPreview: any[], columnsData?: any): string {
let combinedText = '';
// Seguir exatamente o padrão do Python: chamar json_to_free_text para cada objeto separadamente
// 1. metadata_data (dataAsset)
combinedText += this.jsonToFreeText(dataAsset);
// 2. info_data (columns) - usar columnsData se disponível, senão tentar dataAsset.columns
if (columnsData) {
// Adicionar aviso se os dados das colunas são limitados
if (columnsData.columns && columnsData.columns.some((col: any) => col.name?.startsWith('COLUMN_'))) {
combinedText += 'WARNING: Column information is limited or unavailable. Generated column names are placeholders.\n\n';
}
combinedText += this.jsonToFreeText(columnsData);
} else if (dataAsset.columns && Array.isArray(dataAsset.columns)) {
const columnsDataFromAsset = { columns: dataAsset.columns };
combinedText += this.jsonToFreeText(columnsDataFromAsset);
} else {
// Se não há dados das colunas, informar explicitamente
combinedText += 'WARNING: No column metadata available. Cannot generate detailed table structure.\n\n';
}
// 3. preview_data
if (dataPreview && dataPreview.length > 0) {
const previewData = { preview: dataPreview };
combinedText += this.jsonToFreeText(previewData);
}
return combinedText;
}
private jsonToFreeText(jsonObj: any, indentLevel: number = 0): string {
if (!jsonObj || typeof jsonObj !== 'object') {
return '';
}
let text = '';
const indent = ' '.repeat(indentLevel);
// Processa primeiro os dados geográficos se existirem (igual ao Python)
const geoKeys = GEOGRAPHIC_KEYS;
// Adiciona uma seção especial para dados de preview se existirem (igual ao Python)
if ('preview' in jsonObj) {
text += "=== PREVIEW DATA START ===\n";
if (Array.isArray(jsonObj['preview'])) {
// Primeiro, vamos procurar por colunas geográficas
const geoColumns = [];
if (jsonObj['preview'].length > 0 && typeof jsonObj['preview'][0] === 'object') {
for (const key of Object.keys(jsonObj['preview'][0])) {
const keyLower = key.toLowerCase();
if (geoKeys.some(geoTerm => keyLower.includes(geoTerm))) {
geoColumns.push(key);
}
}
}
// Se encontramos colunas geográficas, vamos destacá-las
if (geoColumns.length > 0) {
text += "GEOGRAPHIC DATA FOUND IN COLUMNS:\n";
for (const col of geoColumns) {
text += `=== Column: ${col} ===\n`;
const values = jsonObj['preview'].map(row => String(row[col] || '')).filter(v => v);
text += "Values: " + values.join(", ") + "\n\n";
}
}
// Agora processamos todos os dados normalmente
for (const row of jsonObj['preview']) {
text += this.jsonToFreeText(row, indentLevel + 1);
}
} else {
text += this.jsonToFreeText(jsonObj['preview'], indentLevel + 1);
}
text += "=== PREVIEW DATA END ===\n\n";
}
// Processa o resto dos dados (igual ao Python)
for (const [key, value] of Object.entries(jsonObj)) {
if (key === 'preview') { // Skip preview here since we handled it above
continue;
}
if (typeof value === 'object' && value !== null) {
if (Array.isArray(value)) {
text += `${indent}${this.capitalize(key)}:\n`;
for (const item of value) {
if (typeof item === 'object' && item !== null) {
text += this.jsonToFreeText(item, indentLevel + 1);
} else {
text += `${indent} - ${item}\n`;
}
}
} else {
text += `${indent}${this.capitalize(key)}:\n`;
text += this.jsonToFreeText(value, indentLevel + 1);
}
} else {
// Destaca campos geográficos
if (geoKeys.includes(key.toLowerCase() as any)) {
text += `${indent}!!! GEOGRAPHIC DATA !!! ${this.capitalize(key)}: ${value}\n`;
} else {
text += `${indent}${this.capitalize(key)}: ${value}\n`;
}
}
}
text += '\n';
return text;
}
private capitalize(str: string): string {
return str.charAt(0).toUpperCase() + str.slice(1);
}
private async callAutodriveForDocumentation(combinedText: string, userAccessToken?: string): Promise<string> {
const credentials = this.getAutodriveCredentials();
try {
// Criar dataset temporário com os dados reais
const datasetId = await this.createTemporaryDataset(combinedText, userAccessToken);
// Fazer pergunta ao dataset
const documentation = await this.askQuestionToDataset(datasetId, credentials);
return documentation;
} catch (error) {
this.logger.error(`Error calling Autodrive API: ${error.message}`);
throw new Error(`Autodrive API call failed: ${error.message}`);
}
}
private getAutodriveCredentials(): AutodriveCredentials {
const fallbackCredentials = process.env.AUTODRIVE_KEY || process.env.AUTO_DRIVE_KEY;
// Verificar se as credenciais principais estão disponíveis
if (AUTODRIVE_CONSTANTS.USERNAME && AUTODRIVE_CONSTANTS.PASSWORD) {
return {
username: AUTODRIVE_CONSTANTS.USERNAME,
password: AUTODRIVE_CONSTANTS.PASSWORD,
baseUrl: AUTODRIVE_CONSTANTS.BASE_URL,
model: AUTODRIVE_CONSTANTS.DEFAULT_MODEL,
};
}
// Fallback para credenciais alternativas
if (fallbackCredentials) {
const cleanKey = fallbackCredentials.startsWith("'") && fallbackCredentials.endsWith("'")
? fallbackCredentials.slice(1, -1)
: fallbackCredentials;
return {
username: '',
password: '',
baseUrl: AUTODRIVE_CONSTANTS.BASE_URL,
model: AUTODRIVE_CONSTANTS.DEFAULT_MODEL,
authHeader: `Basic ${cleanKey}`,
};
}
// Fallback de emergência removido por segurança
// Configure as variáveis de ambiente necessárias
throw new Error('No authentication credentials available. Please configure AUTODRIVE_USERNAME and AUTODRIVE_PASSWORD in your .env file');
}
private async askQuestionToDataset(datasetId: string, credentials: AutodriveCredentials): Promise<string> {
const authHeader = credentials.authHeader || this.createAuthHeader(credentials);
const askPayload: AutodriveAskPayload = {
question: AI_DOCUMENTATION_PROMPT.trim(),
fetch_k: AI_DOCUMENTATION_CONFIG.FETCH_K,
k: AI_DOCUMENTATION_CONFIG.K,
model: credentials.model
};
const askHeaders = {
'Authorization': authHeader,
...AUTODRIVE_CONSTANTS.HEADERS
};
const askResponse = await axios.post<AutodriveAskResponse>(
`${credentials.baseUrl}/dataset/${datasetId}/ai_question`,
askPayload,
{
headers: askHeaders,
timeout: AUTODRIVE_CONSTANTS.ASK_TIMEOUT
}
);
// Verificar se a resposta contém a documentação diretamente
if (askResponse.data?.answer) {
const documentation = askResponse.data.answer;
this.logger.info(`Documentation generated successfully - Length: ${documentation.length} characters`);
return documentation;
}
// Se não contém a resposta diretamente, verificar se tem question_id para buscar
if (askResponse.data?.question_id) {
const questionId = askResponse.data.question_id;
// Buscar a resposta da pergunta
const answerResponse = await axios.get<AutodriveAnswerResponse>(
`${credentials.baseUrl}/dataset/${datasetId}/ai_question/${questionId}`,
{
headers: { 'Authorization': authHeader },
timeout: AUTODRIVE_CONSTANTS.ANSWER_TIMEOUT
}
);
// Se a resposta ainda está sendo processada, fazer polling contínuo
if (answerResponse.data?.status === 'started' || !answerResponse.data?.answer) {
let attempts = 0;
const maxAttempts = 12;
const pollInterval = 10000;
while (attempts < maxAttempts) {
attempts++;
await new Promise(resolve => setTimeout(resolve, pollInterval));
try {
const pollResponse = await axios.get<AutodriveAnswerResponse>(
`${credentials.baseUrl}/dataset/${datasetId}/ai_question/${questionId}`,
{
headers: { 'Authorization': authHeader },
timeout: AUTODRIVE_CONSTANTS.ANSWER_TIMEOUT
}
);
// Se a resposta está pronta, retornar
if (pollResponse.data?.answer && pollResponse.data?.status !== 'started') {
const documentation = pollResponse.data.answer;
this.logger.info(`Documentation generated successfully after ${attempts} attempts - Length: ${documentation.length} characters`);
return documentation;
}
// Se ainda está processando, continuar o loop
if (pollResponse.data?.status === 'started') {
continue;
}
// Se falhou, mostrar erro e quebrar
if (pollResponse.data?.status === 'failed') {
throw new Error(`Answer processing failed: ${pollResponse.data.status_reason || 'Unknown error'}`);
}
// Se houve erro ou status inesperado, quebrar o loop
break;
} catch (pollError) {
this.logger.error(`Polling attempt ${attempts} failed: ${pollError.message}`);
continue;
}
}
throw new Error(`Answer still not ready after ${maxAttempts} polling attempts`);
}
if (!answerResponse.data || !answerResponse.data.answer) {
throw new Error(`Answer response invalid: ${JSON.stringify(answerResponse.data)}`);
}
const documentation = answerResponse.data.answer;
this.logger.info(`Documentation generated successfully - Length: ${documentation.length} characters`);
return documentation;
}
throw new Error(`Autodrive API response invalid: ${JSON.stringify(askResponse.data)}`);
}
private async createTemporaryDataset(combinedText: string, userAccessToken?: string): Promise<string> {
if (!AUTODRIVE_CONSTANTS.BASE_URL) {
throw new Error('AUTODRIVE_BASE_URL not configured');
}
if (!combinedText || combinedText.trim().length === 0) {
throw new Error('combinedText is empty - cannot create dataset');
}
const credentials = this.getAutodriveCredentials();
const authHeader = credentials.authHeader || this.createAuthHeader(credentials);
const tempFilePath = await this.createTempFile(combinedText);
try {
const uploadResponse = await this.uploadDataset(tempFilePath, authHeader);
const datasetId = uploadResponse.dataset_id;
await this.waitForDatasetReady(datasetId, authHeader, credentials.baseUrl);
return datasetId;
} catch (error) {
this.logger.error(`Error creating temporary dataset: ${error.message}`);
throw error;
} finally {
await this.cleanupTempFile(tempFilePath);
}
}
private async createTempFile(combinedText: string): Promise<string> {
const tempFileName = `temp_data_${Date.now()}.txt`;
const tempFilePath = `/tmp/${tempFileName}`;
const fs = await import('fs');
fs.writeFileSync(tempFilePath, combinedText, 'utf8');
return tempFilePath;
}
private async uploadDataset(tempFilePath: string, authHeader: string): Promise<AutodriveUploadResponse> {
const tempFileName = tempFilePath.split('/').pop() || 'temp_data.txt';
const fs = await import('fs');
const formData = new FormData();
formData.append('files', fs.createReadStream(tempFilePath), tempFileName);
formData.append('name', tempFileName);
const uploadResponse = await axios.post<AutodriveUploadResponse>(
`${AUTODRIVE_CONSTANTS.BASE_URL}/upload`,
formData,
{
headers: {
'Authorization': authHeader,
...formData.getHeaders(),
},
timeout: 30000, // 30 segundos
}
);
if (!uploadResponse.data?.dataset_id) {
throw new Error('No dataset_id in upload response');
}
return uploadResponse.data;
}
private async waitForDatasetReady(datasetId: string, authHeader: string, autodriveBaseUrl: string): Promise<void> {
const maxAttempts = 12;
const pollInterval = 10000; // 10 segundos
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const statusResponse = await axios.get<DatasetStatusResponse>(
`${autodriveBaseUrl}/dataset/${datasetId}`,
{
headers: { 'Authorization': authHeader },
timeout: 30000,
}
);
if (statusResponse.data?.status === 'success') {
return;
}
if (statusResponse.data?.status === 'failed') {
throw new Error(`Dataset processing failed: ${statusResponse.data.status_reason || 'Unknown error'}`);
}
if (statusResponse.data?.status === 'processing') {
if (attempt < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, pollInterval));
continue;
}
}
if (attempt === maxAttempts) {
throw new Error(`Dataset still not ready after ${maxAttempts} attempts`);
}
} catch (statusError) {
this.logger.error(`Status check error on attempt ${attempt}: ${statusError.message}`);
if (attempt === maxAttempts) {
throw statusError;
}
await new Promise(resolve => setTimeout(resolve, pollInterval));
}
}
}
private async cleanupTempFile(tempFilePath: string | null): Promise<void> {
if (!tempFilePath) return;
try {
const fs = await import('fs');
if (fs.existsSync(tempFilePath)) {
fs.unlinkSync(tempFilePath);
}
} catch (error) {
this.logger.warn(`Failed to remove temporary file: ${error.message}`);
}
}
private createAuthHeader(credentials: AutodriveCredentials): string {
return `Basic ${Buffer.from(`${credentials.username}:${credentials.password}`).toString('base64')}`;
}
}
export { CatalogService };
export { CatalogService };
@@ -1,42 +0,0 @@
/**
* Constantes relacionadas ao Autodrive
*
*/
export const AUTODRIVE_CONSTANTS = {
// URLs e endpoints (apenas do ENV)
BASE_URL: process.env.BASE_URL_AUTODRIVE || process.env.AUTODRIVE_BASE_URL,
// Credenciais (apenas do ENV, sem fallback para segurança)
USERNAME: process.env.AUTODRIVE_USERNAME,
PASSWORD: process.env.AUTODRIVE_PASSWORD,
// Modelo padrão
DEFAULT_MODEL: process.env.AUTODRIVE_MODEL || "gpt-4o",
// Timeouts (em milissegundos)
ASK_TIMEOUT: 120000,
ANSWER_TIMEOUT: 180000,
// Headers
HEADERS: {
'Content-Type': 'application/json',
},
} as const;
/**
* Chaves geográficas para detecção de dados de localização
*/
export const GEOGRAPHIC_KEYS = [
'country', 'countries', 'city', 'cities',
'region', 'regions', 'location', 'state',
'states', 'address'
] as const;
/**
* Países comuns para detecção automática
*/
export const COMMON_COUNTRIES = [
'brazil', 'brasil', 'usa', 'united states',
'canada', 'mexico', 'argentina', 'chile',
'colombia'
] as const;
-8
View File
@@ -320,11 +320,3 @@ export class BatchRemoveRlsRulesRequest {
@ApiPropertyOptional()
id_rls?: string;
}
export type AssetReporter = {
id: string;
display_name: string;
data_asset_type: string;
created_at: string;
tags: string;
}
@@ -1,88 +0,0 @@
/**
*/
export const AI_DOCUMENTATION_PROMPT = `crie uma documentação em Portugues, Ingles e Espanhol seguindo essas instruções
1. Persona: como profissional de governança e engenharia de dados
2. Tarefa: ao receber as informações da tabela criar uma documentação com o seguinte escopo
**A primeira linha do documento tem que conter a seguinte informação: ## Document languages: EN / BR / ES
**A segunda linha tem que obrigatoriamente conter a escrita Table: nome da tabela
**A terceira linha tem que obrigatoriamente conter a escrita Table Schema: nome do table schema
**DIRETRIZ CRUCIAL DE CONSISTÊNCIA E COMPLETUDE DE SCHEMA:**
**1. Fonte Exclusiva de Metadados:** O 'Table Schema' definido na linha acima é a ÚNICA fonte de verdade para o schema dos dados a serem documentados. TODAS as informações subsequentes, especialmente na seção 'Estrutura da Tabela' (incluindo a lista de colunas, seus nomes, tipos de dados, descrições e exemplos) DEVEM ser extraídas EXCLUSIVAMENTE de metadados que correspondem a ESTE 'Table Schema'. Se os dados de entrada que você recebeu contiverem informações para a mesma tabela ou colunas mas de schemas diferentes (ex: um schema 'bronze' e um 'silver'), você DEVE IGNORAR TOTALMENTE as informações dos schemas divergentes para esta tarefa de documentação e utilizar APENAS as do 'Table Schema' aqui especificado.
**2. Listagem Completa de Colunas:** Sua principal tarefa na seção 'Estrutura da Tabela' é identificar e listar TODAS as colunas que pertencem ao 'Table Schema' especificado. Verifique nos dados de entrada fornecidos se há uma indicação explícita do número total de colunas para esta tabela neste schema (por exemplo, um campo como 'Num_columns' ou similar nos metadados da tabela). Você deve se esforçar para listar exatamente essa quantidade de colunas. Se essa contagem não estiver disponível, liste todas as colunas que você puder identificar como pertencentes exclusivamente a este 'Table Schema'. A completude em relação ao schema especificado é essencial.
**Depois de "Estrutura da tablea", incluir a mensagem "Este documento foi gerado por IA", traduzida corretamente para cada idioma.**
**Obrigatoriamente:Após finalizar a versão em Inglês, começar a versão em Português** **Após finalizar a versão em Português, começar a versão em Espanhol** **Antes de começar cada versão, colocar um título como:** - \`## English Version\` (para inglês)
- \`## Versão em Português\` (para português)
- \`## Versión en Español\` (para espanhol)
- Descrição: fornece uma visão geral do ativo de dados,
destacando seu propósito e principal funcionalidade.
Esta sessão resume o conteúdo e o objetivo do ativo, ajudando os usuários a entender rapidamente o que o ativo representa
e como pode ser utilizado em suas análises e decisões.
- Sugestão de Domínio de Dados:
Analise cuidadosamente os dados da tabela e sugira o domínio mais apropriado. Inclua:
- Domínio Sugerido: [Nome do domínio]
- Motivo: [Explicação breve sobre porque a tabela pertence a este domínio]
- Observações: [Qualquer observação adicional relevante]
Exemplos de Domínios de Dados para referência:
- Financeiro: Dados sobre transações, receitas, despesas, etc.
- Recursos Humanos: Dados sobre funcionários, cargos, salários, etc.
- Produtos: Dados sobre produtos, categorias, preços, etc.
- Fornecedores: Dados sobre fornecedores, produtos fornecidos, localizações, etc.
- Marketing: Dados sobre campanhas, leads, conversões, etc.
- Vendas: Dados sobre vendas, clientes, produtos vendidos, etc.
- Operações: Dados sobre processos, logística, produção, etc.
- Clientes: Dados sobre clientes, interações, histórico, etc.
-Tags Sugeridas:
A IA deve gerar tags relevantes **com base nos dados da tabela**.
- **IMPORTANTE: Analise cuidadosamente os dados de preview da tabela (PREVIEW DATA) para encontrar países. Procure em todas as colunas por nomes de países, cidades ou regiões.**
- **Garanta que as tags estejam separadas por espaços vazios, todas na mesma linha, exemplo: #marketing #sales #australia #canada, limitar até 3 países que mais aparecem** - **Os países DEVEM ser extraídos dos dados de preview da tabela. Procure em colunas como City, Country, Region, Location, etc.** - Por que esta tabela é interessante:
Nesta sessão, é destacada a importância do ativo, explicando como ele pode ser útil para os usuários.
São abordadas as formas como o ativo pode melhorar a tomada de decisões, identificar padrões relevantes ou fornecer insights valiosos.
O objetivo é ressaltar a utilidade prática e o impacto positivo que o ativo pode ter em suas atividades.
- Análises potencialmente úteis feitas com esses dados:
Aqui são listadas algumas das análises que podem ser realizadas com o ativo de dados. Inclui sugestões de dashboards,
relatórios ou outros tipos de análises que aproveitam as informações fornecidas pelo ativo.
O objetivo é oferecer maneiras de utilizar os dados para obter insights valiosos e apoiar a tomada de decisões informadas.
- Links Úteis:
Os Links Úteis oferecem recursos adicionais relacionados ao ativo de dados, incluindo guias,
artigos ou outras fontes de informação que podem ajudar os usuários a compreender melhor o ativo e suas aplicações. Além disso,
inclui um link rápido dentro da Dadosfera para ativos relacionados diretamente com o ativo em questão, facilitando a navegação entre os ativos.
- Estrutura da Tabela:
A Estrutura da Tabela detalha TODAS as colunas e os dados disponíveis no ativo, conforme pertencentes ao 'Table Schema' principal definido no início deste documento.
**Instrução Detalhada para Estrutura da Tabela:**
Siga rigorosamente estes passos:
1. Identifique nos dados de entrada (metadados da tabela e das colunas) todas as colunas que pertencem EXCLUSIVAMENTE ao 'Table Schema' especificado no cabeçalho deste documento. Se houver uma contagem de colunas (ex: 'Num_columns') para este schema específico, assegure-se de listar essa quantidade.
2. Para CADA uma dessas colunas identificadas, formate a saída da seguinte maneira, **SEM utilizar NENHUM marcador de lista (como traços ou asteriscos) no início de cada entrada de coluna**. Cada coluna deve ser apresentada como um bloco de texto. Inclua uma linha em branco entre a documentação de cada coluna para separação visual.
- Apresente o NOME_DA_COLUNA em maiúsculas, seguido pelo (TIPO_DE_DADO_EXTRAÍDO_DOS_METADADOS_DO_SCHEMA_CORRETO) entre parênteses.
- O **NOME_DA_COLUNA (TIPO_DE_DADO_EXTRAÍDO_DOS_METADADOS_DO_SCHEMA_CORRETO)** deve estar na primeira linha do bloco da coluna e **inteiramente em negrito**.
- Na linha seguinte, a etiqueta "**Descrição:**" deve estar **em negrito**, seguida pelo texto da descrição da coluna.
- Na linha seguinte à descrição, a etiqueta "**Exemplo:**" deve estar **em negrito**, seguida pelo valor do exemplo. Se o exemplo for um valor literal ou código, formate-o entre crases (\`) se apropriado.
- Se houver informações adicionais relevantes (como "Valores Possíveis:", "Observações:", etc.), coloque a etiqueta correspondente **em negrito** em uma nova linha, seguida pelo seu texto.
Este documento foi gerado por IA.
NOME_COLUNA_1 (TIPO_DADO_SCHEMA_CORRETO_1):
Descrição: [Descrição da coluna 1, do schema correto]
Exemplo: \`[Exemplo de valor para coluna 1, do schema correto]\`
NOME_COLUNA_2 (TIPO_DADO_SCHEMA_CORRETO_2):
Descrição: [Descrição da coluna 2, do schema correto]
Exemplo: \`[Exemplo de valor para coluna 2, do schema correto]\`
(continue este formato com início de cada coluna para TODAS as colunas do 'Table Schema' especificado, garanta com que NUNCA tenha TRAÇO OU PONTO no inicio)
3. Contexto : O usuário ira cadastrar um ativo de dados na nossa plataforma e para ter um bom catalogo ele ira querer gerar a documentação padronizada mas explicativa e
automática
4. Restrições : A documentação deve seguir obrigatoriamente o mesmo padrão principalmente na parte de estrutura de dados
5. Objetivo: O principal objetivo é gerar uma documentação acessível, clara,
automática e padronizada para os usuários que desejem cadastrar um ativo de dados na plataforma`;
/**
* Configurações para a geração de documentação com IA
*/
export const AI_DOCUMENTATION_CONFIG = {
FETCH_K: 250,
K: 100,
} as const;
@@ -1,89 +0,0 @@
import {
Controller,
Get,
Inject,
Param,
Req,
UseFilters,
} from '@nestjs/common';
import {
ApiHeaders,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
IColumnsMetadataResponse,
IDocsResponse,
IPreviewResponse,
} from '../dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ShareService } from './share.service';
import { Request } from 'express';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog/data-asset/share')
@UseFilters(new GrpcToHttpExceptionFilter())
export class ShareController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private catalogShareService: ShareService,
) {
this.logger = dadosferaLogger.logger;
}
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
}
@Get('/:id/preview')
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@Get('/:id/docs')
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
}
-30
View File
@@ -1,30 +0,0 @@
import { Module } from "@nestjs/common";
import { CatalogClientConfiguration } from "../catalog-client";
import { ClientsModule } from "@nestjs/microservices";
import { RolesModule } from "src/modules/roles/roles.module";
import { UsersModule } from "src/modules/users/users.module";
import { CustomersModule } from "src/modules/customers/customers.module";
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@Module({
imports: [
ClientsModule.register([client.providerOptions]),
UsersModule,
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
exports: [ShareModule],
})
export class ShareModule {}
-275
View File
@@ -1,275 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
ProtoServices,
ReadService,
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
import {
ForbiddenException,
Inject,
NotFoundException,
OnModuleInit,
} from '@nestjs/common';
import { CatalogClientConfiguration } from '../catalog-client';
import { ClientGrpc } from '@nestjs/microservices';
import { UsersService } from 'src/modules/users/users.service';
import { RolesService } from 'src/modules/roles/roles.service';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Metadata } from '@grpc/grpc-js';
import { lastValueFrom } from 'rxjs';
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(CatalogClientConfiguration.name)
private readonly grpcClient: ClientGrpc,
private readonly userService: UsersService,
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.catalogReadService =
this.grpcClient.getService<ReadService.CatalogReadServices>(
ProtoServices.CatalogReadServices,
);
}
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { columns_metadata } = await lastValueFrom(
this.catalogReadService.GetDatasetColumnsMetadata(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(columns_metadata);
return result;
}
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { preview } = await lastValueFrom(
this.catalogReadService.GetDatasetPreview(
{ id: shareMetadata.assetId, type: undefined },
metadata,
),
);
const result = JSON.parse(preview);
return result;
}
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id
}))
try {
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
type: isJWT(id) ? 'assigned' : shareMetadata.type,
customer: shareMetadata.customerName
}
if (user) {
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
} else {
await this.mixpanelService.trackShare(request, mixpanelTracker);
}
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { data_asset } = await this.getOneDataAsset({
customer_id: shareMetadata.customerId,
id: shareMetadata.assetId,
metadata,
});
this.logger.info('found asset: ' + JSON.stringify(data_asset));
delete data_asset.p_roles;
delete data_asset.p_users;
data_asset.share_type = 'public';
if (data_asset.share_type !== 'public') throw new NotFoundException();
return { data_asset };
} catch (error) {
this.logger.error(error);
throw error;
}
}
private async getOneDataAsset(data: {
id: string;
customer_id: string;
metadata: Metadata;
}) {
const { customer_id, id, metadata } = data;
const { data_asset } = await lastValueFrom(
this.catalogReadService.GetOneDataAsset(
{ id, type: undefined },
metadata,
),
);
let asset = JSON.parse(data_asset);
asset = {
...asset,
p_roles: asset.roles,
p_users: asset.users,
};
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
return { data_asset: asset[0] };
}
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
});
const { documentation } = await lastValueFrom(
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
);
console.log(documentation);
const docs = JSON.parse(documentation);
return docs;
}
private async getAssetsUsersAndRoles(
data_assets: Array<any>,
customer_id: string,
) {
const { users: customer_users } =
await this.userService.findAllUsersByCustomerId(customer_id);
const { roles: customer_roles } = await this.roleService.roleSearch(
{},
{ customer_id },
);
return data_assets.map((data_asset) => {
const owner = customer_users.find(
(u) => u.id === data_asset.owner,
)?.username;
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
return {
...data_asset,
roles,
users,
owner,
} as typeof data_asset;
});
}
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
if (isJWT(id) && info ){
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
const is_data_manager = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
);
const is_get = user.permissions.includes(
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
);
if (is_data_manager || is_get) {
return info;
}
throw new ForbiddenException(
'You do not have permission to access this data asset.',
);
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}
@@ -1,60 +0,0 @@
/**
* Tipos relacionados à geração de documentação com IA
*/
export interface AutodriveCredentials {
username: string;
password: string;
baseUrl: string;
model: string;
authHeader?: string;
}
export interface AutodriveAskPayload {
question: string;
fetch_k: number;
k: number;
model: string;
}
export interface AutodriveAskResponse {
answer?: string;
question_id?: string;
dataset_id?: string;
}
export interface AutodriveAnswerResponse {
status: 'started' | 'success' | 'failed';
answer?: string;
status_reason?: string;
}
export interface AutodriveUploadResponse {
dataset_id: string;
}
export interface DatasetStatusResponse {
status: 'processing' | 'success' | 'failed';
status_reason?: string;
}
export interface ColumnData {
name: string;
type: string;
description?: string;
nullable?: string;
}
export interface ColumnsMetadata {
columns: ColumnData[];
}
export interface DataPreview {
preview: any[];
}
export interface FormattedDataForAI {
dataAsset: any;
dataPreview: any[];
columnsData?: ColumnsMetadata;
}
@@ -20,7 +20,7 @@ import {
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@Injectable()
export class ConnectionTestService {
@@ -28,7 +28,7 @@ import {
UpdateConnectionDto,
} from './dtos/connection';
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -25,7 +25,7 @@ import { CustomersService } from './customers.service';
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
import { RequestUser, User } from 'src/decorators/user.decorator';
import type { StringValue } from 'ms';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { EnforceMfa } from './dtos/enforce-mfa';
@ApiTags('Customers')
@@ -59,14 +59,6 @@ export class CustomersService implements OnModuleInit {
);
}
async getCustomer(customerId: string) {
return await lastValueFrom(
this.customerService.CustomerFindOneById({
id: customerId
})
)
}
async getLinks(customerId: string) {
try {
const result = await lastValueFrom(
@@ -149,7 +141,6 @@ export class CustomersService implements OnModuleInit {
// const decoded = jwt.decode(jwt_token, { complete: true });
return jwt_token;
}
async getMonitoringDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMonitoringDashboardUrl');
@@ -167,7 +158,6 @@ export class CustomersService implements OnModuleInit {
return res;
}
async getLogsDashboardUrl(metadata: Metadata) {
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
@@ -1,47 +0,0 @@
import { ApiProperty } from "@nestjs/swagger";
export class CreateIdentityProvider {
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
clientSecret: string;
@ApiProperty()
issuerUrl: string;
@ApiProperty()
permissions: number[];
}
export class IdentityProviderResponse {
@ApiProperty()
id: string;
@ApiProperty()
name: string;
@ApiProperty()
clientId: string;
@ApiProperty()
issueUrl: string;
@ApiProperty()
permissions: {
id: number;
name: string;
}[];
}
export class IdentityProviderListResponse {
@ApiProperty()
providers: IdentityProviderResponse[]
}
@@ -1,10 +0,0 @@
export class SsoSignInDto {
readonly nonce: string;
readonly codeVerifier: string;
readonly state: string;
readonly id: string;
readonly clientId: string;
readonly clientSecret: string;
readonly issuerUrl: string;
readonly redirectUrls: string[];
}
@@ -1,246 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
HttpCode,
HttpStatus,
Inject,
Param,
Post,
Put,
Redirect,
Req,
} from '@nestjs/common';
import { IdentityProviderService } from './identity-provider.service';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { ApiOkResponse } from '@nestjs/swagger';
import {
CreateIdentityProvider,
IdentityProviderListResponse,
IdentityProviderResponse,
} from './dto/identity-provider.dto';
import { Request } from 'express';
import ErrorCodes from 'src/utils/errorCodes';
import {
Authenticated,
RequireModule,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
@Controller('identity-providers')
export class IdentityProviderController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private identityProviderService: IdentityProviderService,
) {
this.logger = dadosferaLogger.logger;
}
@Post()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async addIdentityProvider(
@User() user: RequestUser,
@Body() body: CreateIdentityProvider,
@Language() language: LanguageEnum,
) {
this.logger.info('POST /identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
return await this.identityProviderService.create(body, metadata);
}
@Get()
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: IdentityProviderListResponse })
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async getProviders(
@User() user: RequestUser,
@Language() language: LanguageEnum,
) {
this.logger.info('GET identity-providers');
const metadata = PackTheMetadata({
...user,
language,
});
const result = await this.identityProviderService.getList(metadata);
return result;
}
@Delete(':id')
@HttpCode(HttpStatus.NO_CONTENT)
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async deleteIdentityProvider(
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('DELETE /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.deleteIdentityProvider(
id,
metadata,
);
}
@Put(':id')
@HttpCode(HttpStatus.OK)
@Authenticated()
@RequireModule('sso')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async updateIdentityProviders(
@Param('id') id: string,
@Body() body: CreateIdentityProvider,
@User() user: RequestUser,
) {
this.logger.info('PUT /identity-providers');
const metadata = PackTheMetadata({
...user,
});
return await this.identityProviderService.updateIdentityProviders(
id,
body,
metadata,
);
}
@Post('/callback')
@HttpCode(HttpStatus.OK)
async callbackIdp(
@Req() req: Request,
@Language() language: LanguageEnum,
@Body()
body: {
state: string;
code: string;
},
) {
this.logger.info('GET /identity-providers/callback');
const { code, state } = body;
if (!code) {
this.logger.error('No code received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
if (!state) {
this.logger.error('No state received from IDP');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
}
try {
const origin = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + origin);
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${origin}/auth/callback`
: `${origin}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
return await this.identityProviderService.getTokenByIdp(
code,
state,
callbackUrl,
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get('/links')
@HttpCode(HttpStatus.OK)
async providerLinks(@Req() req: Request) {
this.logger.info('GET /identity-providers/links');
try {
const frontDomain = req.headers['origin'] as string;
this.logger.info('Header Origin: ' + frontDomain);
if (!frontDomain) {
this.logger.info('Not found front domain');
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
}
const result =
await this.identityProviderService.identityProvidersLinksPerDomain(
frontDomain,
);
return result;
} catch (error) {
this.logger.error(error);
throw error;
}
}
@Get(':id')
@HttpCode(HttpStatus.OK)
@Redirect()
async loginIdp(
@Param('id') id: string,
@Req() req: Request,
@Language() language: LanguageEnum,
) {
this.logger.info('GET /identity-providers/:id');
try {
const frontDomain =
(req.headers['origin'] as string) || (req.headers['referer'] as string);
this.logger.info(`Front domain: ${frontDomain}`);
const host =
frontDomain.lastIndexOf('/') !== -1
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
: frontDomain;
const lang =
language.substring(0, 2) + language.substring(2).toUpperCase();
const callbackUrl =
process.env.ENV !== 'prd'
? `${host}/auth/callback`
: `${host}/${lang}/auth/callback`;
this.logger.info('Callback URL: ' + callbackUrl);
const redirectUrl =
await this.identityProviderService.loginIdentityProvider(
id,
callbackUrl,
);
this.logger.info(`Redirecting to: ${redirectUrl}`);
return {
url: redirectUrl,
};
} catch (error) {
this.logger.error(error);
throw error;
}
}
}
@@ -1,16 +0,0 @@
import { Module } from '@nestjs/common';
import { IdentityProviderController } from './identity-provider.controller';
import { IdentityProviderService } from './identity-provider.service';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ServicesModule } from 'src/services/service.module';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
controllers: [IdentityProviderController],
providers: [IdentityProviderService, DadosferaLogger],
})
export class IdentityProviderModule {}
@@ -1,185 +0,0 @@
import {
BadRequestException,
Inject,
Injectable,
OnModuleInit,
} from '@nestjs/common';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { Metadata } from '@grpc/grpc-js';
import { Issuer, generators } from 'openid-client';
import { SsoSignInDto } from './dto/sso-signin.dto';
import { CacheService } from 'src/services/cache.service';
import { CreateIdentityProvider } from './dto/identity-provider.dto';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Injectable()
export class IdentityProviderService implements OnModuleInit {
private logger: DadosferaLogger;
private identityProviderService: IdentityProviderProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
private readonly cacheService: CacheService<SsoSignInDto>,
@Inject(DadosferaLogger)
private dadosferaLoggger: DadosferaLogger
) {
this.logger = dadosferaLoggger.logger;
}
onModuleInit() {
this.identityProviderService =
this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
);
}
async create(body: IdentityProviderRequest, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Create")
return await lastValueFrom(
this.identityProviderService.Create(body, metadata),
);
}
async getList(metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC GetList")
return await lastValueFrom(
this.identityProviderService.GetList({}, metadata),
);
}
async loginIdentityProvider(id: string, callbackUrl: string) {
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
const idp = await lastValueFrom(
this.identityProviderService.FindIdentityProvider({ id }),
);
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
const issuer = await Issuer.discover(idp.issuerUrl);
const client = new issuer.Client({
client_id: idp.clientId,
client_secret: idp.clientSecret,
redirect_uris: idp.redirectUrls,
response_types: ['code'],
});
this.logger.info("Generate Challenge")
const code_verifier: string = generators.codeVerifier();
const code_challenge: string = generators.codeChallenge(code_verifier);
this.logger.info("Generate State")
const state = generators.state();
this.logger.info("Generate Nonce")
const nonce = generators.nonce();
// Using state because it is returned in the callback
// and we can use it to retrieve the code_verifier and nonce
this.logger.info("Save Login parameters in redis")
await this.cacheService.set(state, {
codeVerifier: code_verifier,
nonce,
id: idp.id,
state,
clientId: idp.clientId,
clientSecret: idp.clientSecret,
issuerUrl: idp.issuerUrl,
redirectUrls: idp.redirectUrls,
});
this.logger.info("Generate Authorization URL")
const url = client.authorizationUrl({
scope: 'openid email',
response_type: 'code',
code_challenge,
code_challenge_method: 'S256',
state,
nonce,
redirect_uri: callbackUrl,
});
const idpUrl = url + '&identity_provider=' + idp.name;
this.logger.info(idpUrl)
return idpUrl;
}
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
this.logger.info("Get login parameters in redis")
const ssoSign = await this.cacheService.get(state);
if (!ssoSign) {
this.logger.info("Login Parameters Not Found")
throw new BadRequestException('SSO sign-in is expired or not found');
}
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
const issuer = await Issuer.discover(ssoSign.issuerUrl);
const client = new issuer.Client({
client_id: ssoSign.clientId,
client_secret: ssoSign.clientSecret,
redirect_uris: ssoSign.redirectUrls,
});
const params = client.callbackParams(
`${callbackUrl}?code=${code}&state=${state}`,
);
try {
this.logger.info("Get Token Set");
const tokenSet = await client.callback(callbackUrl, params, {
nonce: ssoSign.nonce,
code_verifier: ssoSign.codeVerifier,
state: ssoSign.state
});
this.logger.info("Delete parameters in redis");
await this.cacheService.delete(ssoSign.state);
this.logger.info("Call IdentityProvider GRPC SignInUser");
return await lastValueFrom(
this.identityProviderService.SignInUser({
accessToken: tokenSet.access_token,
idToken: tokenSet.id_token,
refreshToken: tokenSet.refresh_token,
id: ssoSign.id,
}),
);
} catch (error) {
this.logger.error(error);
throw error;
}
}
async deleteIdentityProvider(id: string, metadata: Metadata) {
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
return await lastValueFrom(
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
);
}
async updateIdentityProviders(
id: string,
body: CreateIdentityProvider,
metadata: Metadata,
) {
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
return await lastValueFrom(
this.identityProviderService.UpdateIdentityProvider(
{
id,
...body,
},
metadata,
),
);
}
async identityProvidersLinksPerDomain(frontDomain: string) {
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
return await lastValueFrom(
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
);
}
}
+76 -4
View File
@@ -3,13 +3,14 @@ import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
constructor(
private mixpanelService: MixpanelService
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
) {}
@Post(':id')
async trackEvent(
@@ -19,11 +20,82 @@ export class MixpanelController {
@Req() request
) {
delete body.info;
await this.mixpanelService.track(id, user, request, body)
const mixpanel = init(this.mixpanelToken);
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name
});
await mixpanel.track(id, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return { id, body, user: user.username };
}
capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
+1 -6
View File
@@ -1,8 +1,6 @@
import { Module } from '@nestjs/common';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { MixpanelController } from './mixpanel.controller';
import { MixpanelService } from './mixpanel.service';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@Module({
controllers: [MixpanelController],
@@ -10,12 +8,9 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs';
{
provide: 'MIXPANEL_TOKEN',
useValue: getSecretFromSecretsManager(
`prd/root/mixpanel_token`,
`${process.env.ENV}/root/mixpanel_token`,
),
},
MixpanelService,
DadosferaLogger
],
exports: [MixpanelService]
})
export class MixpanelModule {}
-118
View File
@@ -1,118 +0,0 @@
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { Inject } from '@nestjs/common';
import { Request } from 'express';
import mixpanel, { init } from 'mixpanel';
import { RequestUser } from 'src/decorators/user.decorator';
export class MixpanelService {
logger: DadosferaLogger;
constructor(
@Inject('MIXPANEL_TOKEN')
private readonly mixpanelToken: string,
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
) {
this.logger = dadosferaLogger.logger;
}
async track(eventName: string, user: RequestUser, request: Request, body: any) {
this.logger.info("track: " + JSON.stringify({
eventName,
...body
}))
const mixpanel = init(this.mixpanelToken);
await this.setPeople(user, mixpanel);
await mixpanel.track(eventName, {
distinct_id: user.username,
customer: user.customer_name,
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
async trackShare(request: Request, body: any) {
this.logger.info("trackShare: " + JSON.stringify(body))
const mixpanel = init(this.mixpanelToken);
await mixpanel.track("share_page", {
env: process.env.ENV,
$ip: request.ip,
$os: request.headers['sec-ch-ua-platform'] || '',
$browser: request.headers['user-agent'],
...body,
});
return;
}
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
const separator = user.username.includes('-') ? '-' : '.';
const removeValues = [
'.dadosferatech.dadosfera',
'.demo.dadosfera',
'.dadosferademo',
'.dadosferarh.dadosfera',
'.dadosferatech.dadosfera2',
'.dadosferatech.dadosfera',
'.dadosfera.fin',
'.dadosferafin.dadosfera',
'.praxio.dadosfera',
'.dadosfera.tech',
'.treinamentos@dadosfera.ai',
'.dadosfera2',
'.treinamentosfera',
'.dadosfera',
];
let username = user.username;
removeValues.forEach((value) => {
username = username.replace(value, '');
});
username = username.split('@')?.[0];
username = username.split('+')?.[0];
let firstName = username
.substring(0, username.indexOf(separator))
.replace('dadosfera', '');
let lastName = username
.substring(username.lastIndexOf(separator) + 1)
.replace('dadosfera', '');
if (!firstName) {
firstName = lastName;
lastName = '';
}
firstName = this.capitalize(firstName);
lastName = this.capitalize(lastName);
await mixpanel.people.set(user.username, {
$first_name: firstName,
$last_name: lastName,
$name: this.getFullName(firstName, lastName),
$email: user.username.includes('@')
? user.username
: user.username + '@dadosfera.ai',
customer_name: user.customer_name,
});
}
private capitalize(sentence: string): string {
if (!sentence) {
return '';
}
return sentence[0].toUpperCase() + sentence.substring(1);
}
private getFullName(firstName: string, lastName: string) {
return `${firstName}${lastName ? ' ' + lastName : ''}`;
}
}
@@ -8,7 +8,7 @@ import {
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { NetworkPolicyService } from './network-policy.service';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@Controller('network-policy')
export class NetworkPolicyController {
+1 -1
View File
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
@@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator';
import { OpenDataService } from './open-data.service';
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
import { Metadata } from '@grpc/grpc-js';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { request } from 'http';
import { Request } from 'express';
@@ -45,11 +45,11 @@ export class OpenDataController {
) {
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
// if (!corslist.includes(origin)) {
// this.logger.info('block request by cors list: '+ origin);
// throw new ForbiddenException();
// }
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
if (!corslist.includes(origin)) {
this.logger.info('block request by cors list: '+ origin);
throw new ForbiddenException();
}
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
@@ -32,7 +32,7 @@ import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
import {
@@ -205,10 +205,13 @@ export class PipelinesController {
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.oldPipelinesService.getPipelineStatus(body);
@@ -253,7 +256,6 @@ export class PipelinesController {
});
return res;
});
return result;
}
+15
View File
@@ -20,6 +20,7 @@ import { PermissionsService } from '../permissions/permissions.service';
import { LanguageEnum } from 'src/utils/languages.enum';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
import { PUBLIC_PERMISSIONS_SEQID } from 'src/authentication/permissions.enum';
interface GetRolesPermissionsName {
id: string;
@@ -103,6 +104,20 @@ export class RolesService {
const meta = new Metadata();
meta.add('access_token', access_token);
const { description, name, permissionIds, userIds } = data;
const invalidPermissions = [];
for (const permission of permissionIds) {
if (!PUBLIC_PERMISSIONS_SEQID.includes(permission)) {
invalidPermissions.push(permission);
}
}
if (invalidPermissions.length > 0) {
throw new BadRequestException(
`Invalid permissions: ${invalidPermissions.join(', ')}`,
);
}
const role = await lastValueFrom(
this.rolesClientService.RoleCreate(
{
@@ -1,10 +0,0 @@
export class CreateShareMetadataDto {
assetId: string;
proposedId: string;
type: string;
}
export class UpdateShareMetadataDto {
shareId: string;
type: string;
}
@@ -1,31 +0,0 @@
import { Controller, Get, Body, Param, Put, Patch } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Controller('share')
@Authenticated()
export class ShareMetadataController {
constructor(private readonly shareMetadataService: ShareMetadataService) {}
@Put('/')
findOrcreate(@Body() data: CreateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.create(data, metadata);
}
@Patch('/')
updateShareType(@Body() data: UpdateShareMetadataDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.updateShare(data, metadata);
}
@Get('/:id')
get(@Param('id') id: string, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.shareMetadataService.get(id, metadata);
}
}
@@ -1,16 +0,0 @@
import { Module } from '@nestjs/common';
import { ShareMetadataService } from './share-metadata.service';
import { ShareMetadataController } from './share-metadata.controller';
import { DucClient } from '../duc/client.config';
import { ClientsModule } from '@nestjs/microservices';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [ShareMetadataController],
providers: [ShareMetadataService, DadosferaLogger],
exports: [ShareMetadataService]
})
export class ShareMetadataModule {}
@@ -1,62 +0,0 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Metadata } from '@grpc/grpc-js';
import { ClientGrpc } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
import { ShareMetadataProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CreateShareMetadataDto, UpdateShareMetadataDto } from './dto/create-share-metadata.dto';
@Injectable()
export class ShareMetadataService implements OnModuleInit {
ducService: ShareMetadataProtoService;
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {
this.logger = dadosferaLogger.logger;
}
onModuleInit() {
this.ducService =this.grpcClient.getService<ShareMetadataProtoService>(
ProtoServices.ShareMetadataProtoService,
);
}
create(createShareDto: CreateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.FindOrCreateShareMetadata(
{
assetId: createShareDto.assetId,
proposedId: createShareDto.proposedId,
type: createShareDto.type
},
metadata,
),
);
}
updateShare(updateShare: UpdateShareMetadataDto, metadata: Metadata) {
return lastValueFrom(
this.ducService.ChangeShareMetadataType(
updateShare,
metadata,
),
);
}
get(id: string, metadata: Metadata) {
return lastValueFrom(
this.ducService.GetShareMetadata(
{
id,
},
metadata,
),
);
}
}
-11
View File
@@ -174,14 +174,3 @@ export class GetAllDepartmentsRes {
@ApiProperty()
departments: string[];
}
export interface UserReporter {
name: string;
email: string;
mfaStatus: string;
status: string;
lastLogin: string;
createdAt: string;
updatedAt: string;
}
@@ -8,17 +8,6 @@ import { RolesModule } from '../roles/roles.module';
import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
+1 -24
View File
@@ -12,7 +12,6 @@ import {
Post,
Put,
Query,
Res,
UseFilters,
} from '@nestjs/common';
import {
@@ -25,13 +24,12 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { LanguageEnum } from 'src/utils/languages.enum';
@@ -54,7 +52,6 @@ import {
UpdateUserRes,
} from './dtos/entities';
import { UsersService } from './users.service';
import { Response } from 'express';
@ApiInternalOnlyController()
@ApiTags('Users')
@@ -83,26 +80,6 @@ export class UsersController {
return await this.userService.findAllUsersByCustomerId(user.customer_id);
}
@Get('/download')
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
async downloadUsersInCsv(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@Res() res: Response
) {
this.logger.info('downloadUsersInCsv');
this.userService.setLanguage(language);
const {
file,
filename
} = await this.userService.downloadUsersInCsv(user.customer_id);
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
res.setHeader('Content-Type', 'text/csv');
res.end(file);
}
@Get('hierarchies')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
@ApiOkResponse({ type: GetAllHierarchiesRes })
-11
View File
@@ -9,17 +9,6 @@ import { PermissionsModule } from '../permissions/permissions.module';
// const client = new DucClient();
jest.mock('puppeteer', () => ({
launch: jest.fn().mockResolvedValue({
newPage: jest.fn().mockResolvedValue({
goto: jest.fn(),
evaluate: jest.fn(),
close: jest.fn()
}),
close: jest.fn()
})
}));
const logger = {
info: (...args) => args,
error: (...args) => args,
-30
View File
@@ -21,7 +21,6 @@ import {
IUserByCustomer,
SetUserRolesReq,
UpdateUserReq,
UserReporter,
} from './dtos/entities';
import { RolesService } from '../roles/roles.service';
import { HIERARCHIES } from './hierarchies';
@@ -30,7 +29,6 @@ import { UserByCustomer } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces
import { EnrichErrorCode } from 'src/utils/ErrorBuilder';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
@Injectable()
export class UsersService implements OnModuleInit {
@@ -80,34 +78,6 @@ export class UsersService implements OnModuleInit {
};
}
async downloadUsersInCsv(customerId: string) {
const { users } = await lastValueFrom(
this.usersClientService.UserFindAllByCustomerId({ customerId }),
);
const formatUsers: UserReporter[] = users.map(user => ({
createdAt: user.createdAt,
email: user.email,
lastLogin: user.lastLogin,
mfaStatus: user.mfaStatus,
name: user.name,
status: user.status,
updatedAt: user.updatedAt
}))
const parser = ParserBuilder.build<UserReporter>('csv');
const file = await parser.parse(formatUsers);
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `dadosfera_users_${timestamp}.csv`;
return {
file,
filename
}
}
async findOneById(id: string): Promise<{ user: IUserByCustomer }> {
const { user } = await lastValueFrom(
this.usersClientService.UserFindOneById({ id }),
-23
View File
@@ -1,23 +0,0 @@
import { CACHE_MANAGER, Inject, Injectable } from '@nestjs/common';
import { Cache } from 'cache-manager';
@Injectable()
export class CacheService<T> {
constructor(
@Inject(CACHE_MANAGER)
private readonly cacheManager: Cache,
) {}
async get(key: string): Promise<T | null> {
return await this.cacheManager.get<T>(key);
}
async set(key: string, value: T): Promise<void> {
await this.cacheManager.set(key, value);
}
async delete(key: string) {
await this.cacheManager.del(key);
}
}
-32
View File
@@ -1,32 +0,0 @@
import { CacheModule, Module } from '@nestjs/common';
import { CacheService } from './cache.service';
import { redisStore } from 'cache-manager-ioredis-yet';
@Module({
imports: [
CacheModule.registerAsync({
useFactory: async () => {
const baseRedisConfig = {
ttl: 5 * 1000 * 60, // 5 minute
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT && Number(process.env.REDIS_PORT),
db: process.env.REDIS_DATABASE && Number(process.env.REDIS_DATABASE),
keyPrefix: 'maestro:sso',
}
if (process.env.ENV !== 'local') {
baseRedisConfig['tls'] = {
servername: process.env.REDIS_HOST,
}
}
return {
store: await redisStore(baseRedisConfig),
}
},
}),
],
providers: [CacheService],
exports: [CacheService],
})
export class ServicesModule {}
+2 -2
View File
@@ -22,7 +22,7 @@ export class PDFParser<T> implements Parser<T> {
const html = await this.htmlParser.parse(data);
await page.setContent(html, {
waitUntil: 'networkidle0',
timeout: 90000,
timeout: 30000,
});
// Configurações adicionais para garantir um PDF válido
@@ -43,7 +43,7 @@ export class PDFParser<T> implements Parser<T> {
pageRanges: '',
tagged: true,
outline: false,
timeout: 90000,
timeout: 30000,
});
await browser.close();
+1 -7
View File
@@ -24,21 +24,15 @@ export async function getOauthSecrets() {
const path = process.env.SM_OAUTH_PATH;
const secretsManagerClient = new SecretsManagerClient({});
for (const key in secrets) {
console.log(`Fetching secret for ${key} from path ${path}/${key}`);
const getSecretComand = new GetSecretValueCommand({
SecretId: `${path}/${key}`,
});
const res = await secretsManagerClient
.send(getSecretComand)
.catch((err) => {
console.error(`Error fetching secret for ${key}:`, err);
return null;
});
.catch(() => null);
if (res) {
secrets[key] = JSON.parse(res.SecretString);
console.log(JSON.parse(res.SecretString).client_id)
} else {
console.log("No secret found for", key);
secrets[key] = {
client_id: 'id',
client_secret: 'id',
-5
View File
@@ -75,10 +75,6 @@ const CATALOG = {
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
};
const IDENTITY_PROVIDER = {
INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',
INVALID_HEADER: 'IDENTITY_PROVIDER.INVALID_HEADER',
}
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',
RATE_LIMIT: 'RATE_LIMIT',
@@ -95,7 +91,6 @@ const ErrorCodes = {
TRANSFORMATION,
CONNECTION,
CATALOG,
IDENTITY_PROVIDER
};
export default ErrorCodes;