mirror of
https://github.com/dadosfera/maestro.git
synced 2026-09-04 21:54:48 +00:00
Compare commits
387
Commits
@@ -0,0 +1,12 @@
|
||||
node_modules
|
||||
dist
|
||||
.git
|
||||
*.log
|
||||
npm-debug.log*
|
||||
.DS_Store
|
||||
.env
|
||||
.env.*
|
||||
coverage
|
||||
.nyc_output
|
||||
*.tgz
|
||||
!protospack.tgz
|
||||
@@ -13,11 +13,6 @@ on:
|
||||
options:
|
||||
- stg
|
||||
- prd
|
||||
push_to_dockerhub:
|
||||
description: "Push image to Dockerhub?"
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
extract_environment:
|
||||
@@ -118,23 +113,6 @@ jobs:
|
||||
docker compose -f build.docker-compose.yml build
|
||||
docker compose -f build.docker-compose.yml push
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: ${{inputs.push_to_dockerhub}}
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: dadosfera
|
||||
password: ${{ secrets.DOCKERHUB_PASSWORD }}
|
||||
|
||||
- name: Build, Tag, and Push Image to Dockerhub
|
||||
if: ${{inputs.push_to_dockerhub}}
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
|
||||
ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
|
||||
run: |
|
||||
docker compose -f build.docker-compose.dockerhub.yml build
|
||||
docker compose -f build.docker-compose.dockerhub.yml push
|
||||
|
||||
# - name: Create ZIP file to Deploy AWS Beanstalk
|
||||
# env:
|
||||
# ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
@@ -164,54 +142,11 @@ jobs:
|
||||
docker system prune --volumes -a -f
|
||||
docker system df
|
||||
|
||||
helmfile-deploy:
|
||||
k8s-deploy:
|
||||
needs: [extract_environment, semantic_release, build_ecr_image]
|
||||
runs-on: [self-hosted, "prd-azure"]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v1
|
||||
with:
|
||||
version: 'v3.9.0'
|
||||
|
||||
- name: Install Azure ClI
|
||||
run: |
|
||||
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
|
||||
|
||||
- uses: azure/login@v2
|
||||
with:
|
||||
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
|
||||
- name: Install Helmfile
|
||||
run: |
|
||||
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
|
||||
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
|
||||
mv helmfile /usr/local/bin/
|
||||
helmfile --version
|
||||
|
||||
- name: Install Helm Diff Plugin
|
||||
run: helm plugin install https://github.com/databus23/helm-diff || true
|
||||
|
||||
- name: Setup kubectl
|
||||
uses: azure/setup-kubectl@v1
|
||||
with:
|
||||
version: 'v1.30.1'
|
||||
|
||||
- name: Authenticate with cluster
|
||||
env:
|
||||
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
|
||||
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
|
||||
|
||||
- name: Run Helmfile Apply
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
|
||||
run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
|
||||
uses: ./.github/workflows/k8s-deploy.yml
|
||||
with:
|
||||
cloud: 'oracle'
|
||||
environment: ${{ needs.extract_environment.outputs.environment }}
|
||||
image: ${{ needs.semantic_release.outputs.new_release_version }}
|
||||
secrets: inherit
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
name : K8s deploy
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
cloud:
|
||||
description: "Cloud provider for the deployment"
|
||||
required: true
|
||||
default: "azure"
|
||||
type: string
|
||||
environment:
|
||||
description: "Deployment environment"
|
||||
required: true
|
||||
default: "prd"
|
||||
type: string
|
||||
image:
|
||||
description: "Image Tag"
|
||||
required: true
|
||||
type: string
|
||||
|
||||
jobs:
|
||||
azure:
|
||||
if: inputs.cloud == 'azure'
|
||||
runs-on: [self-hosted, "prd-azure"]
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v1
|
||||
with:
|
||||
version: 'v3.9.0'
|
||||
|
||||
- name: Install Azure ClI
|
||||
run: |
|
||||
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
|
||||
|
||||
- uses: azure/login@v2
|
||||
with:
|
||||
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
|
||||
|
||||
- name: Authenticate with cluster
|
||||
env:
|
||||
CLUSTER_NAME: platform-${{ inputs.environment }}
|
||||
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
|
||||
|
||||
- name: Setup kubectl
|
||||
uses: azure/setup-kubectl@v1
|
||||
with:
|
||||
version: 'v1.30.1'
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
|
||||
- name: Install Helmfile
|
||||
run: |
|
||||
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
|
||||
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
|
||||
sudo mv helmfile /usr/local/bin/
|
||||
helmfile --version
|
||||
|
||||
- name: Install Helm Diff Plugin
|
||||
run: helm plugin install https://github.com/databus23/helm-diff || true
|
||||
|
||||
- name: Run Helmfile Apply
|
||||
env:
|
||||
ENV: ${{ inputs.environment }}
|
||||
IMAGE_TAG: ${{ inputs.image }}
|
||||
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
|
||||
|
||||
oracle:
|
||||
if: inputs.cloud == 'oracle'
|
||||
runs-on: [self-hosted, "prd-oracle"]
|
||||
env:
|
||||
HOME: /home/runner
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v1
|
||||
with:
|
||||
version: 'v3.9.0'
|
||||
|
||||
- name: Install OCI CLI
|
||||
env:
|
||||
HOME: /home/runner
|
||||
run: |
|
||||
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
|
||||
echo "$HOME/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Configure OCI CLI
|
||||
run: |
|
||||
mkdir -p ~/.oci || true
|
||||
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
|
||||
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
|
||||
chmod 600 ~/.oci/oci_api_key.pem
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.8'
|
||||
|
||||
- name: Install Helmfile
|
||||
run: |
|
||||
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
|
||||
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
|
||||
sudo mv helmfile /usr/local/bin/
|
||||
helmfile --version
|
||||
|
||||
- name: Install Helm Diff Plugin
|
||||
run: helm plugin install https://github.com/databus23/helm-diff || true
|
||||
|
||||
- name: Authenticate with OKE cluster
|
||||
env:
|
||||
ENV: ${{ inputs.environment }}
|
||||
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
|
||||
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
|
||||
run: |
|
||||
if [ "$ENV" = "stg" ]; then
|
||||
CLUSTER_ID=$STG_CLUSTER_ID
|
||||
elif [ "$ENV" = "prd" ]; then
|
||||
CLUSTER_ID=$PRD_CLUSTER_ID
|
||||
else
|
||||
echo "Unknown environment: $ENV"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
|
||||
|
||||
- name: Run Helmfile Apply
|
||||
env:
|
||||
ENV: ${{ inputs.environment }}
|
||||
IMAGE_TAG: ${{ inputs.image }}
|
||||
run: helmfile -f deploy/helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG
|
||||
@@ -4,7 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- stg
|
||||
- beta
|
||||
|
||||
jobs:
|
||||
extract_environment:
|
||||
@@ -21,17 +21,13 @@ jobs:
|
||||
fi
|
||||
id: extract_environment
|
||||
|
||||
helmfile-deploy:
|
||||
helmfile-check:
|
||||
env:
|
||||
HOME: /home/runner
|
||||
needs: [extract_environment]
|
||||
runs-on: [self-hosted, "prd-azure"]
|
||||
|
||||
environment: ${{ needs.extract_environment.outputs.environment }}
|
||||
runs-on: [self-hosted, "prd-oracle"]
|
||||
steps:
|
||||
- name: Summary
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
run: |
|
||||
echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
@@ -40,13 +36,28 @@ jobs:
|
||||
with:
|
||||
version: 'v3.9.0'
|
||||
|
||||
- name: Install Azure ClI
|
||||
- name: Determine DNS_HOST based on environment
|
||||
id: set_dns
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
run: |
|
||||
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
|
||||
if [ "$ENV" = "prd" ]; then
|
||||
echo "dns_host=dadosfera.ai" >> $GITHUB_OUTPUT
|
||||
elif [ "$ENV" = "stg" ]; then
|
||||
echo "dns_host=stg.dadosfera.ai" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Install OCI CLI
|
||||
run: |
|
||||
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
|
||||
echo "$HOME/bin" >> $GITHUB_PATH
|
||||
|
||||
- uses: azure/login@v2
|
||||
with:
|
||||
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
|
||||
- name: Configure OCI CLI
|
||||
run: |
|
||||
mkdir -p ~/.oci || true
|
||||
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
|
||||
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
|
||||
chmod 600 ~/.oci/oci_api_key.pem
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
@@ -55,25 +66,40 @@ jobs:
|
||||
|
||||
- name: Install Helmfile
|
||||
run: |
|
||||
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
|
||||
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
|
||||
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
|
||||
mv helmfile /usr/local/bin/
|
||||
sudo mv helmfile /usr/local/bin/
|
||||
helmfile --version
|
||||
|
||||
- name: Install Helm Diff Plugin
|
||||
run: helm plugin install https://github.com/databus23/helm-diff || true
|
||||
- name: Debug Helm env
|
||||
run: |
|
||||
helm env
|
||||
echo "HOME=$HOME"
|
||||
ls -R $HOME/.local/share/helm || true
|
||||
|
||||
- name: Authenticate with OKE cluster
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
|
||||
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
|
||||
run: |
|
||||
if [ "$ENV" = "stg" ]; then
|
||||
CLUSTER_ID=$STG_CLUSTER_ID
|
||||
elif [ "$ENV" = "prd" ]; then
|
||||
CLUSTER_ID=$PRD_CLUSTER_ID
|
||||
else
|
||||
echo "Unknown environment: $ENV"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
|
||||
|
||||
- name: Setup kubectl
|
||||
uses: azure/setup-kubectl@v1
|
||||
with:
|
||||
version: 'v1.30.1'
|
||||
|
||||
- name: Authenticate with cluster
|
||||
env:
|
||||
CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
|
||||
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
|
||||
|
||||
- name: Run Helmfile Diff
|
||||
env:
|
||||
ENV: ${{ needs.extract_environment.outputs.environment }}
|
||||
run: helmfile -f helmfiles/${ENV}.yaml diff
|
||||
run: helmfile -f deploy/helmfiles/${ENV}.yaml diff
|
||||
|
||||
+26
-2
@@ -1,12 +1,23 @@
|
||||
FROM node:18.17-alpine AS base_image
|
||||
FROM node:20-alpine AS base_image
|
||||
RUN npm install -g npm@latest
|
||||
|
||||
FROM base_image AS build_base
|
||||
WORKDIR /app
|
||||
RUN apk update
|
||||
# needed packages to build dependencies from source
|
||||
RUN apk add --no-cache aws-cli
|
||||
RUN apk add --no-cache \
|
||||
aws-cli \
|
||||
chromium \
|
||||
nss \
|
||||
freetype \
|
||||
harfbuzz \
|
||||
ca-certificates \
|
||||
ttf-freefont
|
||||
COPY package*.json ./
|
||||
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
|
||||
|
||||
# run aws cli without mounting secret, because CI already has AWS credentials
|
||||
FROM build_base AS ci_image
|
||||
@@ -40,4 +51,17 @@ WORKDIR /app
|
||||
COPY --from=prod_build /app/dist ./dist
|
||||
COPY --from=prod_build /app/node_modules ./node_modules
|
||||
COPY --from=prod_build /app/package*.json ./
|
||||
RUN apk update
|
||||
# needed packages to build dependencies from source
|
||||
RUN apk add --no-cache \
|
||||
chromium \
|
||||
nss \
|
||||
freetype \
|
||||
harfbuzz \
|
||||
ca-certificates \
|
||||
ttf-freefont
|
||||
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
|
||||
ENTRYPOINT npm run start:prod
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
FROM node:22-alpine AS base_image
|
||||
RUN npm install -g npm@latest
|
||||
|
||||
FROM base_image AS build_base
|
||||
WORKDIR /app
|
||||
RUN apk update
|
||||
RUN apk add --no-cache \
|
||||
aws-cli \
|
||||
chromium \
|
||||
nss \
|
||||
freetype \
|
||||
harfbuzz \
|
||||
ca-certificates \
|
||||
ttf-freefont
|
||||
COPY package*.json ./
|
||||
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
|
||||
|
||||
# Local build with secrets
|
||||
FROM build_base AS build
|
||||
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
||||
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
|
||||
FROM base_image
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/dist ./dist
|
||||
COPY --from=build /app/node_modules ./node_modules
|
||||
COPY --from=build /app/package*.json ./
|
||||
RUN apk update
|
||||
RUN apk add --no-cache \
|
||||
chromium \
|
||||
nss \
|
||||
freetype \
|
||||
harfbuzz \
|
||||
ca-certificates \
|
||||
ttf-freefont
|
||||
|
||||
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
|
||||
ENTRYPOINT ["npm", "run", "start:prod"]
|
||||
@@ -2,7 +2,6 @@
|
||||
<image src="./assets/maestro.svg" style="width:10rem">
|
||||
</p>
|
||||
|
||||
|
||||
# Maestro
|
||||
|
||||
Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços.
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: maestro
|
||||
name: {{ .Values.app_name }}
|
||||
namespace: applications
|
||||
labels:
|
||||
app: maestro
|
||||
app: {{ .Values.app_name }}
|
||||
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: maestro
|
||||
app: {{ .Values.app_name }}
|
||||
|
||||
strategy:
|
||||
rollingUpdate:
|
||||
@@ -20,22 +20,17 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: maestro
|
||||
app: {{ .Values.app_name }}
|
||||
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: {{ .Values.imagePullSecrets }}
|
||||
nodeSelector:
|
||||
"beta.kubernetes.io/os": linux
|
||||
{{- if .Values.affinity }}
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: application
|
||||
operator: In
|
||||
values:
|
||||
- general
|
||||
{{- toYaml .Values.affinity | nindent 8 }}
|
||||
{{- end }}
|
||||
|
||||
tolerations:
|
||||
- key: "kubernetes.azure.com/scalesetpriority"
|
||||
@@ -48,14 +43,14 @@ spec:
|
||||
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
|
||||
ports:
|
||||
- containerPort: {{ .Values.containerPort }}
|
||||
{{- if .Values.resources }}
|
||||
resources:
|
||||
requests:
|
||||
cpu: {{ .Values.resources.requests.cpu }}
|
||||
memory: {{ .Values.resources.requests.memory }}
|
||||
limits:
|
||||
cpu: {{ .Values.resources.limits.cpu }}
|
||||
memory: {{ .Values.resources.limits.memory }}
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
{{- end }}
|
||||
env:
|
||||
# Auth Provider Configuration (cognito or keycloak)
|
||||
- name: AUTH_PROVIDER
|
||||
value: {{ .Values.maestro.auth_provider | default "cognito" | quote }}
|
||||
- name: AWS_IDENTITY_POOL_ID
|
||||
value: {{ .Values.maestro.aws_identity_pool_id }}
|
||||
- name: AWS_REGION
|
||||
@@ -82,6 +77,8 @@ spec:
|
||||
value: "logstash-pipelines.dadosfera.ai"
|
||||
- name: LOGGER_GELF_PORT
|
||||
value: "{{ .Values.maestro.logger_gelf_port }}"
|
||||
- name: LOGGER_CONSOLE_EXTRA
|
||||
value: "true"
|
||||
- name: NIMBUS_BASE_URL
|
||||
value: "http://nimbus-api"
|
||||
- name: NPM_TOKEN
|
||||
@@ -100,6 +97,28 @@ spec:
|
||||
value: {{ .Values.maestro.open_customer_id }}
|
||||
- name: OPEN_GROUP_ID
|
||||
value: {{ .Values.maestro.open_group_id }}
|
||||
- name: DEDICATED_PROXY
|
||||
value: {{ .Values.maestro.dedicated_proxy }}
|
||||
- name: COOKIE_SECRET
|
||||
value: {{ .Values.maestro.cookie_secret }}
|
||||
- name: REDIS_DATABASE
|
||||
value: "{{ .Values.maestro.redis_database }}"
|
||||
- name: REDIS_HOST
|
||||
value: {{ .Values.maestro.redis_host }}
|
||||
- name: REDIS_PORT
|
||||
value: "{{ .Values.maestro.redis_port }}"
|
||||
- name: REDIS_TLS
|
||||
value: "{{ .Values.maestro.redis_tls }}"
|
||||
- name: PLATFORM_API_URL
|
||||
value: {{ .Values.maestro.platform_api_url }}
|
||||
- name: STORAGE_EXPLORER_API_URL
|
||||
value: {{ .Values.maestro.storage_explorer_api_url | quote }}
|
||||
- name: AUTODRIVE_EXTRACTOR_API_URL
|
||||
value: {{ .Values.maestro.autodrive_extractor_api_url | quote }}
|
||||
- name: AUTODRIVE_CORE_API_URL
|
||||
value: {{ .Values.maestro.autodrive_core_api_url | quote }}
|
||||
- name: AUTODRIVE_ASSISTANT_API_URL
|
||||
value: {{ .Values.maestro.autodrive_assistant_api_url | quote }}
|
||||
- name: JWT_PRIVATE_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -108,15 +127,26 @@ spec:
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: prd-maestro
|
||||
name: prd-{{ .Values.app_name }}
|
||||
key: AWS_ACCESS_KEY_ID
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: prd-maestro
|
||||
name: prd-{{ .Values.app_name }}
|
||||
key: AWS_SECRET_ACCESS_KEY
|
||||
- name: AWS_DEFAULT_REGION
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: prd-maestro
|
||||
name: prd-{{ .Values.app_name }}
|
||||
key: AWS_DEFAULT_REGION
|
||||
# Elasticsearch
|
||||
- name: ELASTICSEARCH_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: prd-{{ .Values.app_name }}
|
||||
key: ELASTICSEARCH_URL
|
||||
- name: ELASTICSEARCH_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: prd-{{ .Values.app_name }}
|
||||
key: ELASTICSEARCH_API_KEY
|
||||
@@ -0,0 +1,41 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/server-snippet: |
|
||||
underscores_in_headers on;
|
||||
ignore_invalid_headers on;
|
||||
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
|
||||
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
|
||||
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
|
||||
{{- if .Values.maestro.restricted_ip}}
|
||||
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
|
||||
{{- end }}
|
||||
|
||||
generation: 1
|
||||
labels:
|
||||
app: {{ .Values.app_name }}
|
||||
{{- if .Values.maestro.dedicated_proxy}}
|
||||
name: open-data-{{ .Values.app_name }}
|
||||
{{- else }}
|
||||
name: open-data
|
||||
{{- end }}
|
||||
namespace: applications
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: {{ .Values.hostname }}
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: {{ .Values.app_name }}
|
||||
port:
|
||||
number: {{ .Values.ingress.port }}
|
||||
path: /open-data/sharing-ocean-data
|
||||
pathType: Prefix
|
||||
@@ -0,0 +1,36 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
|
||||
nginx.ingress.kubernetes.io/server-snippet: |
|
||||
underscores_in_headers on;
|
||||
ignore_invalid_headers on;
|
||||
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
|
||||
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
|
||||
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
|
||||
{{- if .Values.maestro.restricted_ip}}
|
||||
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
|
||||
{{- end }}
|
||||
|
||||
generation: 1
|
||||
labels:
|
||||
app: {{ .Values.app_name }}
|
||||
name: {{ .Values.app_name }}
|
||||
namespace: applications
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: {{ .Values.hostname }}
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: {{ .Values.app_name }}
|
||||
port:
|
||||
number: {{ .Values.ingress.port }}
|
||||
path: /
|
||||
pathType: Prefix
|
||||
@@ -1,17 +1,17 @@
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: prd-maestro
|
||||
name: prd-{{ .Values.app_name }}
|
||||
namespace: applications
|
||||
labels:
|
||||
app: maestro
|
||||
app: {{ .Values.app_name }}
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
name: secretsmanager-prd
|
||||
kind: SecretStore
|
||||
target:
|
||||
name: prd-maestro
|
||||
name: prd-{{ .Values.app_name }}
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: AWS_ACCESS_KEY_ID
|
||||
@@ -38,3 +38,15 @@ spec:
|
||||
version: "AWSCURRENT"
|
||||
property: token
|
||||
|
||||
- secretKey: ELASTICSEARCH_URL
|
||||
remoteRef:
|
||||
key: {{ .Values.maestro.env }}/microservices/elasticsearch
|
||||
version: "AWSCURRENT"
|
||||
property: ELASTICSEARCH_URL
|
||||
|
||||
- secretKey: ELASTICSEARCH_API_KEY
|
||||
remoteRef:
|
||||
key: {{ .Values.maestro.env }}/microservices/elasticsearch
|
||||
version: "AWSCURRENT"
|
||||
property: ELASTICSEARCH_API_KEY
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: maestro
|
||||
name: {{ .Values.app_name }}
|
||||
namespace: applications
|
||||
labels:
|
||||
app: maestro
|
||||
app: {{ .Values.app_name }}
|
||||
|
||||
spec:
|
||||
type: ClusterIP
|
||||
ports:
|
||||
- name: maestro
|
||||
- name: {{ .Values.app_name }}
|
||||
protocol: TCP
|
||||
port: {{ .Values.service.port }}
|
||||
targetPort: {{ .Values.service.targetPort }}
|
||||
selector:
|
||||
app: maestro
|
||||
app: {{ .Values.app_name }}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
maestro:
|
||||
env: stg
|
||||
duc_url: duc.stg.dadosfera.ai
|
||||
pi_factory_url: pi-factory.stg.dadosfera.ai
|
||||
in_factory_url: in-factory.stg.dadosfera.ai
|
||||
tr_factory_url: in-factory.stg.dadosfera.ai
|
||||
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
|
||||
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
|
||||
redis_database: "1"
|
||||
platform_api_url: https://xs2hkhq07k.execute-api.us-east-1.amazonaws.com
|
||||
storage_explorer_api_url: "http://storage-explorer-{customer}.data-apps.svc.cluster.local:8000/api"
|
||||
|
||||
hostname: maestro.stg.dadosfera.ai
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
affinity: null
|
||||
@@ -9,6 +9,7 @@ image:
|
||||
pullPolicy: IfNotPresent
|
||||
# Overrides the image tag whose default is the chart appVersion.
|
||||
tag: 1.56.0
|
||||
app_name: maestro
|
||||
containerPort: 3333
|
||||
imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
|
||||
service:
|
||||
@@ -26,6 +27,9 @@ resources:
|
||||
cpu: 2000m
|
||||
memory: 2Gi
|
||||
maestro:
|
||||
# Auth provider: "cognito" (default) or "keycloak"
|
||||
# Note: maestro doesn't connect to Keycloak directly, only duc does
|
||||
auth_provider: "cognito"
|
||||
aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
|
||||
duc_url: duc.dadosfera.ai
|
||||
in_factory_url: in-factory.dadosfera.ai
|
||||
@@ -42,9 +46,31 @@ maestro:
|
||||
upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
|
||||
open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e
|
||||
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
|
||||
platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com
|
||||
storage_explorer_api_url: "https://storage-explorer-{customer}.dadosfera.ai/api"
|
||||
autodrive_extractor_api_url: "https://autodrive-extractor-api-{customer}.dadosfera.ai"
|
||||
autodrive_core_api_url: "https://autodrive-api-{customer}.dadosfera.ai"
|
||||
autodrive_assistant_api_url: "https://autodrive-assistant-api-{customer}.dadosfera.ai"
|
||||
dedicated_proxy: ""
|
||||
restricted_ip: ""
|
||||
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
|
||||
redis_port: "6379"
|
||||
redis_database: "0"
|
||||
redis_tls: "true"
|
||||
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
targetCPUUtilizationPercentage: 80
|
||||
targetMemoryUtilizationPercentage: 80
|
||||
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: name
|
||||
operator: In
|
||||
values:
|
||||
- product
|
||||
@@ -0,0 +1,56 @@
|
||||
releases:
|
||||
- name: maestro
|
||||
chart: ../helm-chart
|
||||
values:
|
||||
- ../helm-chart/values.yaml
|
||||
set:
|
||||
- name: app_name
|
||||
value: maestro
|
||||
- name: maestro.duc_url
|
||||
value: duc.dadosfera.ai
|
||||
- name: hostname
|
||||
value: maestro.dadosfera.ai
|
||||
- name: maestro.pi_factory_url
|
||||
value: pi-factory.dadosfera.ai
|
||||
- name: maestro.in_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.tr_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.open_customer_id
|
||||
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||
- name: maestro.open_group_id
|
||||
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
|
||||
- name: replicaCount
|
||||
value: 2
|
||||
|
||||
- name: unimed-maestro
|
||||
chart: ../helm-chart
|
||||
values:
|
||||
- ../helm-chart/values.yaml
|
||||
set:
|
||||
- name: app_name
|
||||
value: maestro-unimed
|
||||
- name: maestro.duc_url
|
||||
value: duc.dadosfera.ai
|
||||
- name: hostname
|
||||
value: maestro-unimed.dadosfera.ai
|
||||
- name: maestro.pi_factory_url
|
||||
value: pi-factory.dadosfera.ai
|
||||
- name: maestro.in_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.tr_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.open_customer_id
|
||||
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||
- name: maestro.open_group_id
|
||||
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
|
||||
# Customer id
|
||||
- name: maestro.dedicated_proxy
|
||||
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
|
||||
- name: replicaCount
|
||||
value: 1
|
||||
# 10.70.0.0/16 internal network
|
||||
# 137.131.167.254/32 loadbalancer
|
||||
# 159.112.184.81/32 cluster ip for the uptime request ingest
|
||||
- name: maestro.restricted_ip
|
||||
value: "177.52.172.0/24, 189.84.160.157/32, 186.237.171.146/32, 137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
|
||||
@@ -0,0 +1,30 @@
|
||||
charts:
|
||||
- name: maestro
|
||||
chart: ../helm-chart
|
||||
values:
|
||||
- ../helm-chart/values.yaml
|
||||
- ../helm-chart/values-stg.yaml
|
||||
|
||||
|
||||
# Environment to test Network Policies
|
||||
- name: private-maestro
|
||||
chart: ../helm-chart
|
||||
values:
|
||||
- ../helm-chart/values.yaml
|
||||
- ../helm-chart/values-stg.yaml
|
||||
set:
|
||||
- name: app_name
|
||||
value: maestro-private
|
||||
- name: hostname
|
||||
value: private-maestro.stg.dadosfera.ai
|
||||
# Customer id
|
||||
- name: maestro.dedicated_proxy
|
||||
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
|
||||
- name: replicaCount
|
||||
value: 1
|
||||
- name: affinity
|
||||
value: null
|
||||
- name: resources
|
||||
value: null
|
||||
- name: maestro.restricted_ip
|
||||
value: "137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
|
||||
+3334
-560
File diff suppressed because it is too large
Load Diff
Vendored
+3
@@ -14,6 +14,9 @@ declare global {
|
||||
AWS_REGION: string;
|
||||
OPEN_GROUP_ID: string;
|
||||
OPEN_CUSTOMER_ID: string;
|
||||
DEDICATED_PROXY: string;
|
||||
COOKIE_SECRET: string;
|
||||
REDIS_TLS?: string;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
charts:
|
||||
- name: maestro
|
||||
chart: ../maestro
|
||||
values:
|
||||
- ../maestro/values.yaml
|
||||
set:
|
||||
- name: maestro.duc_url
|
||||
value: duc.dadosfera.ai
|
||||
- name: hostname
|
||||
value: maestro.dadosfera.ai
|
||||
- name: maestro.pi_factory_url
|
||||
value: pi-factory.dadosfera.ai
|
||||
- name: maestro.in_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.tr_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.open_customer_id
|
||||
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||
- name: maestro.open_group_id
|
||||
value: c0afdcce-c5be-40d0-9d1d-2d271121f14a
|
||||
- name: replicaCount
|
||||
value: 2
|
||||
@@ -1,22 +0,0 @@
|
||||
charts:
|
||||
- name: maestro
|
||||
chart: ../maestro
|
||||
values:
|
||||
- ../maestro/values.yaml
|
||||
set:
|
||||
- name: maestro.duc_url
|
||||
value: duc.stg.dadosfera.ai
|
||||
- name: hostname
|
||||
value: maestro.stg.dadosfera.ai
|
||||
- name: maestro.pi_factory_url
|
||||
value: pi-factory.dadosfera.ai
|
||||
- name: maestro.in_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.tr_factory_url
|
||||
value: in-factory.dadosfera.ai
|
||||
- name: maestro.open_customer_id
|
||||
value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
|
||||
- name: maestro.open_group_id
|
||||
value: e3f98a2f-7748-4981-8505-7695c8ca8218
|
||||
- name: replicaCount
|
||||
value: 1
|
||||
@@ -1,23 +0,0 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
@@ -1,28 +0,0 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
nginx.ingress.kubernetes.io/server-snippet: |
|
||||
underscores_in_headers on;
|
||||
ignore_invalid_headers on;
|
||||
|
||||
generation: 1
|
||||
labels:
|
||||
app: maestro
|
||||
name: open-data
|
||||
namespace: applications
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: {{ .Values.hostname }}
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: maestro
|
||||
port:
|
||||
number: {{ .Values.ingress.port }}
|
||||
path: /open-data/sharing-ocean-data
|
||||
pathType: Prefix
|
||||
@@ -1,27 +0,0 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
nginx.ingress.kubernetes.io/server-snippet: |
|
||||
underscores_in_headers on;
|
||||
ignore_invalid_headers on;
|
||||
|
||||
generation: 1
|
||||
labels:
|
||||
app: maestro
|
||||
name: maestro
|
||||
namespace: applications
|
||||
spec:
|
||||
ingressClassName: nginx
|
||||
rules:
|
||||
- host: {{ .Values.hostname }}
|
||||
http:
|
||||
paths:
|
||||
- backend:
|
||||
service:
|
||||
name: maestro
|
||||
port:
|
||||
number: {{ .Values.ingress.port }}
|
||||
path: /
|
||||
pathType: Prefix
|
||||
@@ -4,6 +4,7 @@
|
||||
"compilerOptions": {
|
||||
"assets": [
|
||||
"**/*.proto",
|
||||
"assets/**/*",
|
||||
{
|
||||
"include": "i18n/**/*",
|
||||
"watchAssets": true
|
||||
|
||||
Generated
+3447
-1633
File diff suppressed because it is too large
Load Diff
+25
-3
@@ -27,10 +27,15 @@
|
||||
"test:e2e": "jest --config ./test/jest-e2e.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-crypto/sha256-js": "^5.2.0",
|
||||
"@aws-sdk/client-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/client-secrets-manager": "^3.414.0",
|
||||
"@aws-sdk/credential-provider-node": "^3.940.0",
|
||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||
"@aws-sdk/signature-v4": "^3.370.0",
|
||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||
"@dadosfera/protospack": "2.5.3",
|
||||
"@dadosfera/protospack-v2": "3.37.0-beta.22",
|
||||
"@dadosfera/protospack-v2": "3.38.0-beta.28",
|
||||
"@grpc/grpc-js": "^1.9.3",
|
||||
"@grpc/proto-loader": "^0.7.9",
|
||||
"@nestjs/cli": "^9.5.0",
|
||||
@@ -44,33 +49,50 @@
|
||||
"@nestjs/schematics": "^9.2.0",
|
||||
"@nestjs/swagger": "^6.3.0",
|
||||
"@nestjs/testing": "^9.4.3",
|
||||
"axios": "^0.27.2",
|
||||
"axios": "^0.30.2",
|
||||
"cache-manager": "^5.1.4",
|
||||
"cache-manager-ioredis-yet": "^1.1.0",
|
||||
"class-transformer": "^0.5.1",
|
||||
"class-validator": "^0.14.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cron-parser": "^4.9.0",
|
||||
"csv": "^6.3.11",
|
||||
"dotenv": "^14.3.2",
|
||||
"elastic-apm-node": "^3.50.0",
|
||||
"handlebars": "^4.7.8",
|
||||
"helmet": "^5.1.1",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"jwk-to-pem": "^2.0.5",
|
||||
"mixpanel": "^0.17.0",
|
||||
"ms": "^3.0.0-canary.1",
|
||||
"openid-client": "^5.7.1",
|
||||
"passport": "^0.6.0",
|
||||
"passport-facebook": "^3.0.0",
|
||||
"passport-forcedotcom": "^0.2.0",
|
||||
"passport-google-oauth20": "^2.0.0",
|
||||
"passport-hubspot-oauth2": "^1.0.3",
|
||||
"passport-mailchimp": "^1.1.0",
|
||||
"puppeteer": "^24.7.2",
|
||||
"redis": "^4.5.1",
|
||||
"reflect-metadata": "^0.1.13",
|
||||
"rimraf": "^3.0.2",
|
||||
"rxjs": "^7.5.5",
|
||||
"swagger-ui-express": "^4.6.3"
|
||||
},
|
||||
"overrides": {
|
||||
"multer": "1.4.5-lts.1"
|
||||
"multer": "2.0.2",
|
||||
"form-data": "^4.0.4",
|
||||
"body-parser": "^1.20.3",
|
||||
"cross-spawn": "^7.0.5",
|
||||
"glob": "^10.5.0",
|
||||
"path-to-regexp": "^3.3.0",
|
||||
"semver": "^7.5.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cache-manager": "^4.0.6",
|
||||
"@types/cookie-parser": "^1.4.9",
|
||||
"@types/express": "^4.17.17",
|
||||
"@types/express-session": "^1.18.1",
|
||||
"@types/jest": "27.0.2",
|
||||
"@types/jsonwebtoken": "^8.5.9",
|
||||
"@types/jwk-to-pem": "^2.0.1",
|
||||
|
||||
+21
-1
@@ -28,7 +28,16 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
|
||||
import { CustomersModule } from './modules/customers/customers.module';
|
||||
import { OpenDataModule } from './modules/open-data/open-data.module';
|
||||
import { ThemeModule } from './modules/theme/theme.module';
|
||||
import { IdentityProviderModule } from './modules/identity-provider/identity-provider.module';
|
||||
import { NetworkPolicyModule } from './modules/network-policy/network-policy.module';
|
||||
import { AssignModule } from './modules/assign/assign.module';
|
||||
import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module';
|
||||
import { ApiKeyModule } from './modules/api-key/api-key.module';
|
||||
import { PlatformApiModule } from './modules/platform-api/platform-api.module';
|
||||
import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module';
|
||||
import { AutodriveExtractorModule } from './modules/autodrive-extractor/autodrive-extractor.module';
|
||||
import { AutodriveCoreModule } from './modules/autodrive-core/autodrive-core.module';
|
||||
import { AutodriveAssistantModule } from './modules/autodrive-assistant/autodrive-assistant.module';
|
||||
|
||||
@Module({
|
||||
providers: [
|
||||
@@ -62,9 +71,20 @@ import { NetworkPolicyModule } from './modules/network-policy/network-policy.mod
|
||||
CustomersModule,
|
||||
OpenDataModule,
|
||||
ThemeModule,
|
||||
NetworkPolicyModule,
|
||||
AssignModule,
|
||||
ShareMetadataModule,
|
||||
NetworkPolicyModule,
|
||||
ApiKeyModule,
|
||||
IdentityProviderModule,
|
||||
NetworkPolicyModule,
|
||||
PlatformApiModule,
|
||||
StorageExplorerModule,
|
||||
AutodriveExtractorModule,
|
||||
AutodriveCoreModule,
|
||||
AutodriveAssistantModule,
|
||||
//Always leave HealthModule last, so it is on the bottom of swagger
|
||||
HealthModule,
|
||||
NetworkPolicyModule
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="pt-br">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Dadosfera Relatório de PII</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Quicksand:wght@400;500;700&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
@page {
|
||||
size: A4 landscape; /* Alterado para paisagem (landscape) */
|
||||
margin: 15mm 10mm; /* Reduzido para proporcionar mais espaço */
|
||||
}
|
||||
body {
|
||||
font-family: 'Quicksand', sans-serif;
|
||||
color: #5c5c5c;
|
||||
margin: 0;
|
||||
padding: 10px;
|
||||
font-size: 12px; /* Reduzindo o tamanho da fonte */
|
||||
}
|
||||
.container {
|
||||
margin: 0;
|
||||
width: 100%;
|
||||
}
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.logo {
|
||||
max-width: 150px; /* Reduzida para economizar espaço */
|
||||
height: auto;
|
||||
}
|
||||
h1 {
|
||||
color: #0d003b;
|
||||
font-weight: 700;
|
||||
margin-left: 20px;
|
||||
font-size: 24px; /* Tamanho ajustado */
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-top: 15px;
|
||||
table-layout: fixed; /* Importante: define larguras fixas */
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
border: 1px solid #d0d0d0;
|
||||
}
|
||||
th {
|
||||
background-color: #1700a2;
|
||||
color: white;
|
||||
font-weight: bold;
|
||||
text-align: left;
|
||||
padding: 8px 10px;
|
||||
border: 1px solid #3a26b8;
|
||||
font-size: 11px; /* Tamanho ajustado */
|
||||
word-wrap: break-word; /* Permite quebra de palavras */
|
||||
overflow-wrap: break-word;
|
||||
}
|
||||
td {
|
||||
padding: 6px 10px;
|
||||
border: 1px solid #d0d0d0;
|
||||
font-size: 11px; /* Tamanho ajustado */
|
||||
word-wrap: break-word; /* Permite quebra de palavras */
|
||||
overflow-wrap: break-word;
|
||||
}
|
||||
/* Definindo larguras específicas para cada coluna */
|
||||
th:nth-child(1), td:nth-child(1) { width: 14%; } /* Database */
|
||||
th:nth-child(2), td:nth-child(2) { width: 14%; } /* Schema */
|
||||
th:nth-child(3), td:nth-child(3) { width: 17%; } /* Tabela */
|
||||
th:nth-child(4), td:nth-child(4) { width: 17%; } /* Coluna */
|
||||
th:nth-child(5), td:nth-child(5) { width: 13%; } /* Tipo de Dado */
|
||||
th:nth-child(6), td:nth-child(6) { width: 25%; } /* Regras PII */
|
||||
|
||||
tr:nth-child(even) {
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
tr:nth-child(odd) {
|
||||
background-color: white;
|
||||
}
|
||||
.info-section {
|
||||
margin-top: 20px;
|
||||
color: #5c5c5c;
|
||||
}
|
||||
.timestamp {
|
||||
font-style: italic;
|
||||
text-align: right;
|
||||
margin-top: 15px;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<img src="https://dadosfera.ai/wp-content/webp-express/webp-images/uploads/2022/06/Logo-Dadosfera1-1.png.webp" alt="Logo Dadosfera" class="logo">
|
||||
<h1>Relatório de PII</h1>
|
||||
</div>
|
||||
|
||||
<div class="info-section">
|
||||
<p>Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.</p>
|
||||
</div>
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Database</th>
|
||||
<th>Schema</th>
|
||||
<th>Tabela</th>
|
||||
<th>Coluna</th>
|
||||
<th>Tipo de Dado</th>
|
||||
<th>Regras PII</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{{#each dados}}
|
||||
<tr>
|
||||
<td>{{database_name}}</td>
|
||||
<td>{{table_schema}}</td>
|
||||
<td>{{table_name}}</td>
|
||||
<td>{{column_name}}</td>
|
||||
<td>{{data_type}}</td>
|
||||
<td>{{pii_rules}}</td>
|
||||
</tr>
|
||||
{{/each}}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<p class="timestamp">Gerado em: {{dataGeracao}}</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -17,6 +17,7 @@ import { PERMISSIONS_GROUPS } from './permissions.enum';
|
||||
import { AuthClientService } from '../modules/auth/auth.service';
|
||||
|
||||
import ErrorCodes from '../utils/errorCodes';
|
||||
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
|
||||
|
||||
const logger = {
|
||||
info: (...args) => args,
|
||||
@@ -99,6 +100,7 @@ describe('authentication.guard', () => {
|
||||
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
|
||||
customer_name: 'dadosfera',
|
||||
customer_tier: 'BASIC',
|
||||
customer_modules: []
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
@@ -120,6 +122,12 @@ describe('authentication.guard', () => {
|
||||
provide: APP_GUARD,
|
||||
useClass: AuthenticationGuard,
|
||||
},
|
||||
{
|
||||
provide: ApiKeyService,
|
||||
useValue: {
|
||||
get: () => Promise.resolve(null)
|
||||
}
|
||||
}
|
||||
],
|
||||
controllers: [NoClassAuthController, ClassAuthConditionController],
|
||||
}).compile();
|
||||
@@ -440,18 +448,18 @@ describe('authentication.guard', () => {
|
||||
NoClassAuthTest(null, null);
|
||||
ClassAuthConditionTest(null, null);
|
||||
|
||||
const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
|
||||
NoClassAuthTest(tokenZ, ['zendesk']);
|
||||
ClassAuthConditionTest(tokenZ, ['zendesk']);
|
||||
// const tokenZ = CreateToken([PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN]);
|
||||
// NoClassAuthTest(tokenZ, ['zendesk']);
|
||||
// ClassAuthConditionTest(tokenZ, ['zendesk']);
|
||||
|
||||
const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
|
||||
NoClassAuthTest(tokenM, ['metabase']);
|
||||
ClassAuthConditionTest(tokenM, ['metabase']);
|
||||
// const tokenM = CreateToken([PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE]);
|
||||
// NoClassAuthTest(tokenM, ['metabase']);
|
||||
// ClassAuthConditionTest(tokenM, ['metabase']);
|
||||
|
||||
const tokenZM = CreateToken([
|
||||
PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
|
||||
PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
|
||||
]);
|
||||
NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
|
||||
ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
|
||||
// const tokenZM = CreateToken([
|
||||
// PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
|
||||
// PERMISSIONS_GROUPS.DATAVIZ.permissions.METABASE,
|
||||
// ]);
|
||||
// NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
|
||||
// ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
OnApplicationBootstrap,
|
||||
ExecutionContext,
|
||||
Inject,
|
||||
ForbiddenException,
|
||||
} from '@nestjs/common';
|
||||
import { Reflector } from '@nestjs/core';
|
||||
import assert from 'assert';
|
||||
@@ -17,6 +18,7 @@ import {
|
||||
import { RequestUser } from '../decorators/user.decorator';
|
||||
import ErrorBuilder from '../utils/ErrorBuilder';
|
||||
import ErrorCodes from '../utils/errorCodes';
|
||||
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
|
||||
|
||||
@Injectable()
|
||||
export class AuthenticationGuard
|
||||
@@ -31,6 +33,7 @@ export class AuthenticationGuard
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private reflector: Reflector,
|
||||
private authClient: AuthClientService,
|
||||
private apiKeyService: ApiKeyService
|
||||
) {
|
||||
this.pems = new Map();
|
||||
this.logger = dadosferaLogger.logger;
|
||||
@@ -48,20 +51,40 @@ export class AuthenticationGuard
|
||||
});
|
||||
}
|
||||
|
||||
canActivate(ctx: ExecutionContext): boolean {
|
||||
async canActivate(ctx: ExecutionContext): Promise<boolean> {
|
||||
const authFunctions = this.reflector.getAllAndMerge<
|
||||
AuthenticationFunction[]
|
||||
>(AUTH_FUNCTION_KEY, [ctx.getClass(), ctx.getHandler()]);
|
||||
const mustBeAuthenticated = authFunctions.length > 0;
|
||||
|
||||
const request = ctx.switchToHttp().getRequest();
|
||||
const accessToken = this.validateToken(request, mustBeAuthenticated);
|
||||
|
||||
if (!mustBeAuthenticated) {
|
||||
// no need to be authenticated
|
||||
return true;
|
||||
}
|
||||
|
||||
const request = ctx.switchToHttp().getRequest();
|
||||
const apiKey = request.get('X-api-key');
|
||||
if (apiKey) {
|
||||
const {
|
||||
api_key
|
||||
} = await this.apiKeyService.get(apiKey);
|
||||
|
||||
request.user = {
|
||||
user_id: api_key.user_id,
|
||||
username: api_key.username,
|
||||
permissions: api_key.permissions,
|
||||
customer_id: api_key.customer_id,
|
||||
customer_name: api_key.customer_name,
|
||||
customer_tier: api_key.customer_tier,
|
||||
customer_modules: api_key.customer_modules,
|
||||
access_token: apiKey,
|
||||
};
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
const accessToken = this.validateToken(request, mustBeAuthenticated);
|
||||
|
||||
if (!accessToken) {
|
||||
// couldn't load valid token
|
||||
throw new ErrorBuilder(ErrorCodes.AUTH.UNAUTHORIZED);
|
||||
@@ -113,6 +136,18 @@ export class AuthenticationGuard
|
||||
return false;
|
||||
}
|
||||
|
||||
// Bloquear outros customer de usar o maestor dedicado
|
||||
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
|
||||
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== accessTokenPayload.customer_id) {
|
||||
throw new ErrorBuilder(ErrorCodes.AUTH.FORBIDDEN);
|
||||
}
|
||||
|
||||
// Bloquear o customer de acesso o maestro publico
|
||||
const hasNetworkPolicyModule = accessTokenPayload.customer_modules.includes('network-policy');
|
||||
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
|
||||
throw new ForbiddenException(ErrorCodes.AUTH.FORBIDDEN);
|
||||
}
|
||||
|
||||
request.accessTokenPayload = accessTokenPayload;
|
||||
request.user = {
|
||||
user_id: accessTokenPayload.user_id,
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import jwt, { JwtPayload } from 'jsonwebtoken';
|
||||
|
||||
export function extractUserFrom(aRawJwt: string) {
|
||||
const decodedToken = jwt.decode(aRawJwt, {
|
||||
complete: true,
|
||||
});
|
||||
|
||||
const payload = decodedToken.payload as JwtPayload;
|
||||
|
||||
return {
|
||||
user_id: payload.user_id,
|
||||
username: payload.username,
|
||||
permissions: payload.permissions,
|
||||
customer_id: payload.customer_id,
|
||||
customer_name: payload.customer_name,
|
||||
customer_tier: payload.customer_tier,
|
||||
customer_modules: payload.customer_modules,
|
||||
access_token: aRawJwt,
|
||||
}
|
||||
}
|
||||
@@ -116,6 +116,44 @@ export const PERMISSIONS_GROUPS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
IMPORT_FILES: {
|
||||
title: {
|
||||
'pt-br': 'Coletar | Importar arquivos',
|
||||
'en-us': 'Collect | Import files',
|
||||
'es-es': 'Colecta | Importar archivos',
|
||||
},
|
||||
permissions: {
|
||||
VIEW: {
|
||||
seqid: 48,
|
||||
claim: 'import-file:view',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Importar arquivos',
|
||||
'en-us': 'Import files',
|
||||
'es-es': 'Importar archivos',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AI_CHAT: {
|
||||
title: {
|
||||
'pt-br': 'AutodriveDDF',
|
||||
'en-us': 'AutodriveDDF',
|
||||
'es-es': 'AutodriveDDF',
|
||||
},
|
||||
permissions: {
|
||||
VIEW: {
|
||||
seqid: 49,
|
||||
claim: 'ai-chat:view',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'AutodriveDDF',
|
||||
'en-us': 'AutodriveDDF',
|
||||
'es-es': 'AutodriveDDF',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
CONNECTION: {
|
||||
title: {
|
||||
'pt-br': 'Coletar | Fontes de dados',
|
||||
@@ -340,16 +378,6 @@ export const PERMISSIONS_GROUPS = {
|
||||
'es-es': 'Gestor de catálogos. Puede ver y editar todos los activos.',
|
||||
},
|
||||
},
|
||||
EMBED_ANALYTICS: {
|
||||
seqid: 44,
|
||||
claim: 'catalog:embed',
|
||||
usage: PermissionUsages.INTERNAL,
|
||||
name: {
|
||||
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
|
||||
'en-us': 'Access Embedding analytics Module',
|
||||
'es-es': 'Acceder al Módulo de Incorporación de Activos',
|
||||
},
|
||||
},
|
||||
TRIGGER_CATALOG_TASK: {
|
||||
seqid: 45,
|
||||
claim: 'catalog:trigger-task',
|
||||
@@ -362,6 +390,44 @@ export const PERMISSIONS_GROUPS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
LINEAGE: {
|
||||
title: {
|
||||
'pt-br': 'Explorar | Linhagem',
|
||||
'en-us': 'Explore | Lineage',
|
||||
'es-es': 'Explorar | Linaje',
|
||||
},
|
||||
permissions: {
|
||||
VIEW: {
|
||||
seqid: 50,
|
||||
claim: 'lineage:view',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Acessar ao módulo de Linhagem',
|
||||
'en-us': 'Access to Lineage module',
|
||||
'es-es': 'Acceda al módulo de Linaje',
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
EMBED: {
|
||||
title: {
|
||||
'pt-br': 'Analisar | Incorporação',
|
||||
'en-us': 'Analyze | Embedding',
|
||||
'es-es': 'Analizar | Incorporación',
|
||||
},
|
||||
permissions: {
|
||||
EMBED_ANALYTICS: {
|
||||
seqid: 44,
|
||||
claim: 'catalog:embed',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Acessar Módulo de Incorporação de Ativos',
|
||||
'en-us': 'Access Embedding analytics Module',
|
||||
'es-es': 'Acceder al Módulo de Incorporación de Activos',
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
CONNECTORS: {
|
||||
title: {
|
||||
'pt-br': 'Conectores',
|
||||
@@ -602,6 +668,122 @@ export const PERMISSIONS_GROUPS = {
|
||||
},
|
||||
},
|
||||
},
|
||||
STORAGE_EXPLORER: {
|
||||
title: {
|
||||
'pt-br': 'Storage Explorer',
|
||||
'en-us': 'Storage Explorer',
|
||||
'es-es': 'Storage Explorer',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 51,
|
||||
claim: 'storage-explorer:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Storage Explorer',
|
||||
'en-us': 'Read Storage Explorer data',
|
||||
'es-es': 'Leer datos del Storage Explorer',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 52,
|
||||
claim: 'storage-explorer:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Storage Explorer',
|
||||
'en-us': 'Write Storage Explorer data',
|
||||
'es-es': 'Escribir datos en Storage Explorer',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_EXTRACTOR: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Extractor',
|
||||
'en-us': 'Autodrive Extractor',
|
||||
'es-es': 'Autodrive Extractor',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 53,
|
||||
claim: 'autodrive-extractor:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Extractor',
|
||||
'en-us': 'Read Autodrive Extractor data',
|
||||
'es-es': 'Leer datos del Autodrive Extractor',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 54,
|
||||
claim: 'autodrive-extractor:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Extractor',
|
||||
'en-us': 'Write Autodrive Extractor data',
|
||||
'es-es': 'Escribir datos en Autodrive Extractor',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_CORE: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Core',
|
||||
'en-us': 'Autodrive Core',
|
||||
'es-es': 'Autodrive Core',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 55,
|
||||
claim: 'autodrive-core:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Core',
|
||||
'en-us': 'Read Autodrive Core data',
|
||||
'es-es': 'Leer datos del Autodrive Core',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 56,
|
||||
claim: 'autodrive-core:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Core',
|
||||
'en-us': 'Write Autodrive Core data',
|
||||
'es-es': 'Escribir datos en Autodrive Core',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AUTODRIVE_ASSISTANT: {
|
||||
title: {
|
||||
'pt-br': 'Autodrive Assistant',
|
||||
'en-us': 'Autodrive Assistant',
|
||||
'es-es': 'Autodrive Assistant',
|
||||
},
|
||||
permissions: {
|
||||
READ: {
|
||||
seqid: 57,
|
||||
claim: 'autodrive-assistant:read',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Ler dados do Autodrive Assistant',
|
||||
'en-us': 'Read Autodrive Assistant data',
|
||||
'es-es': 'Leer datos del Autodrive Assistant',
|
||||
},
|
||||
},
|
||||
WRITE: {
|
||||
seqid: 58,
|
||||
claim: 'autodrive-assistant:write',
|
||||
usage: PermissionUsages.PUBLIC,
|
||||
name: {
|
||||
'pt-br': 'Escrever dados no Autodrive Assistant',
|
||||
'en-us': 'Write Autodrive Assistant data',
|
||||
'es-es': 'Escribir datos en Autodrive Assistant',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
export interface DadosferaModule {
|
||||
name: string;
|
||||
@@ -613,7 +795,10 @@ export interface DadosferaModule {
|
||||
export const DADOSFERA_MODULES_KEYS = {
|
||||
LOG_DASHBOARD: 'logs-dashboard',
|
||||
ACCESS_DASHBOARD: 'access-dashboard',
|
||||
DANGER_ZONE: 'danger-zone'
|
||||
DANGER_ZONE: 'danger-zone',
|
||||
PII: 'pii',
|
||||
EMBED: 'embedded-analytics',
|
||||
EMBED_ASSIGNED: 'embed-assigned',
|
||||
}
|
||||
|
||||
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
|
||||
|
||||
@@ -9,6 +9,7 @@ import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
|
||||
import { AuthClientService } from '../modules/auth/auth.service';
|
||||
import ErrorCodes from '../utils/errorCodes';
|
||||
import { User } from './user.decorator';
|
||||
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
|
||||
|
||||
const logger = {
|
||||
info: (...args) => args,
|
||||
@@ -52,6 +53,7 @@ describe('user.decorator', () => {
|
||||
customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',
|
||||
customer_name: 'dadosfera',
|
||||
customer_tier: 'BASIC',
|
||||
customer_modules: [],
|
||||
access_token: '',
|
||||
};
|
||||
|
||||
@@ -74,6 +76,12 @@ describe('user.decorator', () => {
|
||||
provide: APP_GUARD,
|
||||
useClass: AuthenticationGuard,
|
||||
},
|
||||
{
|
||||
provide: ApiKeyService,
|
||||
useValue: {
|
||||
get: () => Promise.resolve(null)
|
||||
}
|
||||
}
|
||||
],
|
||||
controllers: [UserController],
|
||||
}).compile();
|
||||
@@ -175,5 +183,5 @@ describe('user.decorator', () => {
|
||||
|
||||
const token = CreateToken();
|
||||
fakeUserPayload.access_token = token;
|
||||
UserTest(token);
|
||||
// UserTest(token);
|
||||
});
|
||||
|
||||
+31
-1
@@ -3,11 +3,14 @@ import { NestFactory } from '@nestjs/core';
|
||||
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
||||
import helmet from 'helmet';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { json, urlencoded } from 'express';
|
||||
|
||||
import { AppModule } from './app.module';
|
||||
import { writeFileSync } from 'fs';
|
||||
import { execSync } from 'child_process';
|
||||
import { INestApplication } from '@nestjs/common';
|
||||
import cookieParser from 'cookie-parser';
|
||||
|
||||
async function bootstrap() {
|
||||
DadosferaLogger.setupLogger({
|
||||
serviceName: 'maestro',
|
||||
@@ -15,16 +18,43 @@ async function bootstrap() {
|
||||
});
|
||||
const logger = new DadosferaLogger();
|
||||
|
||||
const corsOrigins = [];
|
||||
|
||||
if (process.env.ENV === 'local') {
|
||||
corsOrigins.push('http://localhost:4200');
|
||||
} else {
|
||||
corsOrigins.push(
|
||||
'https://app.stg.dadosfera.ai',
|
||||
'https://app.dadosfera.ai',
|
||||
'https://private-frontend.stg.dadosfera.ai',
|
||||
'https://unimed.dadosfera.ai',
|
||||
'https://boston-scientific.dadosfera.ai',
|
||||
'https://plataforma.dadosfera.ai'
|
||||
);
|
||||
}
|
||||
|
||||
const app = await NestFactory.create(AppModule, {
|
||||
logger,
|
||||
cors: {
|
||||
origin: '*',
|
||||
origin: corsOrigins,
|
||||
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
|
||||
preflightContinue: false,
|
||||
optionsSuccessStatus: 204,
|
||||
credentials: true,
|
||||
},
|
||||
});
|
||||
|
||||
app.use(helmet());
|
||||
app.use(cookieParser(process.env.COOKIE_SECRET));
|
||||
|
||||
if (process.env.ENV !== 'local') {
|
||||
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
|
||||
app.use(
|
||||
'/catalog/register-dataset',
|
||||
urlencoded({ extended: true, limit: '10mb' }),
|
||||
);
|
||||
}
|
||||
|
||||
configureSwagger(app);
|
||||
await app.listen(3333);
|
||||
if (process.env.KILL_AFTER_START) await app.close();
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { Controller, Get, Post, Body, Param, Delete, UseFilters, Inject } from '@nestjs/common';
|
||||
import { ApiKeyService } from './api-key.service';
|
||||
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
|
||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||
import { ApiHeaders, ApiTags, ApiResponse } from '@nestjs/swagger';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Controller('api-key')
|
||||
@Authenticated()
|
||||
@ApiTags('ApiKey')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||
export class ApiKeyController {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private readonly apiKeyService: ApiKeyService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Post()
|
||||
@ApiResponse({ type: CreateApiKeyResponseDto })
|
||||
async create(@Body() createApiKeyDto: CreateApiKeyDto, @User() user: RequestUser): Promise<CreateApiKeyResponseDto> {
|
||||
this.logger.info('POST /api-key', {
|
||||
permissions: createApiKeyDto.permissions,
|
||||
method: 'create'
|
||||
});
|
||||
const result = await this.apiKeyService.create(createApiKeyDto, user);
|
||||
this.logger.info('POST /api-key success', {
|
||||
id: result.id,
|
||||
method: 'create'
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
@Get()
|
||||
@ApiResponse({ type: [ApiKeyBaseResponseDto] })
|
||||
async findAll(@User() user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
|
||||
this.logger.info('GET /api-key', {
|
||||
method: 'findAll'
|
||||
});
|
||||
const result = await this.apiKeyService.findAll(user);
|
||||
this.logger.info('GET /api-key success', {
|
||||
count: result.length,
|
||||
method: 'findAll'
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
async remove(@Param('id') id: string, @User() user: RequestUser): Promise<void> {
|
||||
this.logger.info('DELETE /api-key/:id', {
|
||||
id,
|
||||
method: 'remove'
|
||||
});
|
||||
await this.apiKeyService.remove(id, user);
|
||||
this.logger.info('DELETE /api-key/:id success', {
|
||||
id,
|
||||
method: 'remove'
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ApiKeyService } from './api-key.service';
|
||||
import { ApiKeyController } from './api-key.controller';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
const ducClient = new DucClient();
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ClientsModule.register([ducClient.providerOptions])
|
||||
],
|
||||
controllers: [ApiKeyController],
|
||||
providers: [ApiKeyService, DadosferaLogger],
|
||||
exports: [ApiKeyService]
|
||||
})
|
||||
export class ApiKeyModule {}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { Injectable, Inject, OnModuleInit } from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { CreateApiKeyDto, CreateApiKeyResponseDto, ApiKeyBaseResponseDto } from './dto/api-key.dto';
|
||||
import { RequestUser } from 'src/decorators/user.decorator';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { ApiKeyWriteProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
|
||||
@Injectable()
|
||||
export class ApiKeyService implements OnModuleInit {
|
||||
private apiKeyService: ApiKeyWriteProtoService;
|
||||
|
||||
constructor(
|
||||
@Inject(DucClient.name) private readonly client: ClientGrpc,
|
||||
) {}
|
||||
|
||||
onModuleInit() {
|
||||
this.apiKeyService = this.client.getService<ApiKeyWriteProtoService>(ProtoServices.ApiKeyWriteProtoService);
|
||||
}
|
||||
|
||||
create(createApiKeyDto: CreateApiKeyDto, user: RequestUser): Promise<CreateApiKeyResponseDto> {
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
return lastValueFrom(this.apiKeyService.CreateApiKey({
|
||||
permissions: createApiKeyDto.permissions
|
||||
}, metadata));
|
||||
}
|
||||
|
||||
async findAll(user: RequestUser): Promise<ApiKeyBaseResponseDto[]> {
|
||||
const metadata = PackTheMetadata(user);
|
||||
console.log(metadata)
|
||||
|
||||
const data = await lastValueFrom(this.apiKeyService.ListApiKeys({}, metadata));
|
||||
return data.api_keys;
|
||||
}
|
||||
|
||||
async remove(id: string, user: RequestUser) {
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
await lastValueFrom(this.apiKeyService.DeleteApiKey({ id }, metadata));
|
||||
}
|
||||
|
||||
async get(key: string) {
|
||||
const metadata = PackTheMetadata({});
|
||||
|
||||
return await lastValueFrom(this.apiKeyService.GetApiKey({ key }, metadata));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsArray, IsNumber } from 'class-validator';
|
||||
|
||||
export class PermissionDto {
|
||||
@ApiProperty({ type: Number })
|
||||
id: number;
|
||||
|
||||
@ApiProperty({ type: String })
|
||||
name: string;
|
||||
}
|
||||
|
||||
export class ApiKeyBaseResponseDto {
|
||||
@ApiProperty({ type: String, format: 'uuid' })
|
||||
id: string;
|
||||
|
||||
@ApiProperty({ type: String })
|
||||
key_mask: string;
|
||||
|
||||
@ApiProperty({ type: [PermissionDto] })
|
||||
permissions: PermissionDto[];
|
||||
|
||||
@ApiProperty({ type: String, format: 'date-time' })
|
||||
created_at: string;
|
||||
|
||||
@ApiProperty({ type: String })
|
||||
created_by: string;
|
||||
}
|
||||
|
||||
export class CreateApiKeyResponseDto extends ApiKeyBaseResponseDto {
|
||||
@ApiProperty({ type: String })
|
||||
key: string;
|
||||
}
|
||||
|
||||
export class CreateApiKeyDto {
|
||||
@ApiProperty({ type: [Number], description: 'Array of permission IDs' })
|
||||
@IsArray()
|
||||
@IsNumber({}, { each: true })
|
||||
permissions: number[];
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common';
|
||||
import { AssignService } from './assign.service';
|
||||
import { CreateAssignDto } from './dto/create-assign.dto';
|
||||
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
|
||||
@Controller('assign')
|
||||
@Authenticated()
|
||||
export class AssignController {
|
||||
constructor(private readonly assignService: AssignService) {}
|
||||
|
||||
@Put('/public-key')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
|
||||
)
|
||||
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
|
||||
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
|
||||
const metadata = PackTheMetadata(user);
|
||||
return this.assignService.create(createAssignDto, metadata);
|
||||
}
|
||||
|
||||
@Get('/public-key')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
|
||||
)
|
||||
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
|
||||
async get(@User() user: RequestUser) {
|
||||
const metadata = PackTheMetadata(user);
|
||||
try {
|
||||
return await this.assignService.get(metadata);
|
||||
} catch (error) {
|
||||
throw new NotFoundException(error.message)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { AssignService } from './assign.service';
|
||||
import { AssignController } from './assign.controller';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
|
||||
const client = new DucClient();
|
||||
|
||||
@Module({
|
||||
imports: [ClientsModule.register([client.providerOptions])],
|
||||
controllers: [AssignController],
|
||||
providers: [AssignService, DadosferaLogger]
|
||||
})
|
||||
export class AssignModule {}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import { CreateAssignDto } from './dto/create-assign.dto';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { DucClient } from 'src/modules/duc/client.config';
|
||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||
import { lastValueFrom } from 'rxjs';import { AssingProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
|
||||
@Injectable()
|
||||
export class AssignService implements OnModuleInit {
|
||||
|
||||
ducService: AssingProtoService;
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
this.ducService =this.grpcClient.getService<AssingProtoService>(
|
||||
ProtoServices.AssingProtoService,
|
||||
);
|
||||
}
|
||||
|
||||
async create(createAssignDto: CreateAssignDto, metadata: Metadata) {
|
||||
const data = await lastValueFrom(this.ducService.CreateOrUpdateAssignPublicKey(createAssignDto, metadata))
|
||||
return data;
|
||||
}
|
||||
|
||||
async get(metadata: Metadata) {
|
||||
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export class CreateAssignDto {
|
||||
publicKey: string;
|
||||
}
|
||||
@@ -12,6 +12,8 @@ import {
|
||||
Redirect,
|
||||
Req,
|
||||
Param,
|
||||
Res,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiHeaders,
|
||||
@@ -26,7 +28,7 @@ import {
|
||||
AuthConfirmResetPasswordRequest,
|
||||
AuthEnableTotpMfaRequest,
|
||||
AuthDisableTotpMfaRequest,
|
||||
AuthVerifyTotpMfaRequest,
|
||||
AuthVerifyTotpMfaRequest
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
|
||||
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||
@@ -45,14 +47,21 @@ import {
|
||||
AuthSignInRes,
|
||||
BulkEditRequest,
|
||||
} from './dtos/login';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { AuthGuard } from '@nestjs/passport';
|
||||
import { Request } from 'express';
|
||||
import { Request, Response } from 'express';
|
||||
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import jwt, { JwtPayload } from 'jsonwebtoken';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
|
||||
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
|
||||
|
||||
type CookiesValues = {
|
||||
accessToken?: string;
|
||||
refreshToken?: string;
|
||||
userId?: string
|
||||
}
|
||||
|
||||
@ApiTags('Auth')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@@ -66,6 +75,7 @@ export class AuthController {
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private authClient: AuthClientService,
|
||||
private apiKeyService: ApiKeyService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
|
||||
@@ -86,13 +96,46 @@ export class AuthController {
|
||||
async signIn(
|
||||
@Body() { username, password, totp }: AuthSignInReq,
|
||||
@Language() language: LanguageEnum,
|
||||
@Headers('origin') origin: string,
|
||||
): Promise<AuthSignInRes> {
|
||||
this.logger.info('/auth - SignIn');
|
||||
const frontHost = origin.replace(/^https?:\/\//, '');
|
||||
const metadata = PackTheMetadata({ language, custom_host: frontHost });
|
||||
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
|
||||
return this.authClient.signIn({ username, password, totp }, metadata);
|
||||
@Res() res: Response,
|
||||
) {
|
||||
try {
|
||||
this.logger.info('/auth - SignIn');
|
||||
const metadata = PackTheMetadata({ language });
|
||||
this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON()));
|
||||
const data = await this.authClient.signIn({ username, password, totp }, metadata);
|
||||
|
||||
if (data.tokens) {
|
||||
this.authClient.writeAuthSession(res, {
|
||||
accessToken: data.tokens.accessToken,
|
||||
refreshToken: data.tokens.refreshToken,
|
||||
userId: data.user.id
|
||||
});
|
||||
}
|
||||
|
||||
return res.send(data);
|
||||
} catch (error) {
|
||||
this.logger.error('/auth - SignIn - ERROR', error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Post('sign-out')
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
async signOut(
|
||||
@Language() language: LanguageEnum,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
try {
|
||||
this.logger.info('/auth - SignOut');
|
||||
|
||||
this.authClient.cleanUpAuthSession(res);
|
||||
|
||||
return res.send();
|
||||
} catch (error) {
|
||||
this.logger.error('/auth - SignIn - ERROR', error);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Post('refresh-access-token')
|
||||
@@ -102,17 +145,26 @@ export class AuthController {
|
||||
@Body() body: AuthRefreshAccessTokenReq,
|
||||
@Language() language: LanguageEnum,
|
||||
@Headers('origin') origin: string,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
this.logger.info('/auth - RefreshAccessToken');
|
||||
const { refreshToken, customerName: customer_name } = body;
|
||||
const frontHost = origin.replace(/^https?:\/\//, '');
|
||||
const { refreshToken, userId } = body;
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
language,
|
||||
customer_name,
|
||||
custom_host: frontHost,
|
||||
});
|
||||
|
||||
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
|
||||
const data = await this.authClient.refreshAccessToken({ refreshToken, userId }, metadata);
|
||||
|
||||
this.authClient.writeAuthSession(res, {
|
||||
accessToken: data.accessToken,
|
||||
refreshToken: data.refreshToken,
|
||||
userId
|
||||
});
|
||||
|
||||
return res.send(data);
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@@ -140,13 +192,14 @@ export class AuthController {
|
||||
) {
|
||||
this.logger.info('/auth - change-password');
|
||||
|
||||
const { oldPassword, newPassword } = body;
|
||||
const { oldPassword, newPassword, totpCode } = body;
|
||||
const { authorization: accessToken } = headers;
|
||||
|
||||
return this.authClient.changePassword({
|
||||
accessToken,
|
||||
oldPassword,
|
||||
newPassword,
|
||||
totpCode,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -164,7 +217,8 @@ export class AuthController {
|
||||
|
||||
const { username } = body;
|
||||
|
||||
return this.authClient.resetPassword({ username }, metadata);
|
||||
await this.authClient.resetPassword({ username }, metadata);
|
||||
return { authProvider: process.env.AUTH_PROVIDER || 'cognito' };
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@@ -420,4 +474,59 @@ export class AuthController {
|
||||
|
||||
return this.authClient.resetUsers(body.users, metadata);
|
||||
}
|
||||
|
||||
@Get('me')
|
||||
async getMe(@Req() req: Request, @Res() res: Response) {
|
||||
this.logger.info('GET /auth/me ')
|
||||
|
||||
// Check for API key header first
|
||||
const apiKey = req.get('X-Api-key');
|
||||
if (apiKey) {
|
||||
this.logger.info('Authenticating via X-Api-key header');
|
||||
const { api_key } = await this.apiKeyService.get(apiKey);
|
||||
|
||||
const userDto = {
|
||||
id: api_key.user_id,
|
||||
name: api_key.username,
|
||||
customer: {
|
||||
id: api_key.customer_id,
|
||||
name: api_key.customer_name,
|
||||
tier: api_key.customer_tier,
|
||||
}
|
||||
};
|
||||
|
||||
return res.status(200).json(userDto);
|
||||
}
|
||||
|
||||
// Get token and headers
|
||||
const accessToken = req.cookies['ddf-auth'];
|
||||
const refreshToken = req.cookies['ddf-refresh-auth'];
|
||||
const userId = req.cookies['ddf-user-id'];
|
||||
const resourceHost = req.headers["host"]
|
||||
|
||||
const hasUserSession = Boolean(accessToken) && Boolean(userId);
|
||||
this.logger.info('Has User Session: ' + hasUserSession);
|
||||
|
||||
if (!hasUserSession) {
|
||||
throw new UnauthorizedException()
|
||||
}
|
||||
|
||||
try {
|
||||
const userDto = await this.authClient.validateUserSession(accessToken, resourceHost);
|
||||
return res.status(200).json(userDto);
|
||||
} catch (error) {
|
||||
|
||||
if (!refreshToken) {
|
||||
this.logger.error('Invalid refresh token or customer name');
|
||||
throw new UnauthorizedException("Invalid refresh token or customer name");
|
||||
};
|
||||
|
||||
const {
|
||||
authSession,
|
||||
user
|
||||
} = await this.authClient.refreshUserSession(refreshToken, userId, resourceHost);
|
||||
this.authClient.writeAuthSession(res, authSession);
|
||||
return res.status(200).json(user);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,10 +8,11 @@ import { AuthClientService } from './auth.service';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { GoogleLoginStrategy } from './passport-strategies/google-strategy';
|
||||
import { getOauthSecrets } from 'src/utils/OauthSecrets';
|
||||
import { ApiKeyModule } from '../api-key/api-key.module';
|
||||
const client = new DucClient();
|
||||
|
||||
@Module({
|
||||
imports: [ClientsModule.register([client.providerOptions])],
|
||||
imports: [ClientsModule.register([client.providerOptions]), ApiKeyModule],
|
||||
controllers: [AuthController],
|
||||
providers: [
|
||||
AuthClientService,
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
import { OnModuleInit, Inject, Injectable } from '@nestjs/common';
|
||||
import {
|
||||
OnModuleInit,
|
||||
Inject,
|
||||
Injectable,
|
||||
ForbiddenException,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
|
||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||
import { AuthProtoService as AuthServiceInterface } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
import {
|
||||
AuthProtoService as AuthServiceInterface,
|
||||
UsersProtoService,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
import {
|
||||
AuthSnowflakeSignInRequest,
|
||||
AuthSignInRequest,
|
||||
@@ -17,16 +27,28 @@ import {
|
||||
AuthResetPasswordRequest,
|
||||
AuthVerifyResetPasswordCodeRequest,
|
||||
AuthConfirmResetPasswordRequest,
|
||||
AuthSignInResponse,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { BulkEditResponse } from './dtos/login';
|
||||
import { BulkEditResponse, UserDTO } from './dtos/login';
|
||||
import jwt, { JwtPayload } from 'jsonwebtoken';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { Request, Response } from 'express';
|
||||
|
||||
type AuthSession = {
|
||||
accessToken?: string;
|
||||
refreshToken?: string;
|
||||
userId?: string;
|
||||
};
|
||||
|
||||
@Injectable()
|
||||
export class AuthClientService implements OnModuleInit {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
private authService: AuthServiceInterface;
|
||||
private userService: UsersProtoService;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
@@ -39,6 +61,10 @@ export class AuthClientService implements OnModuleInit {
|
||||
this.authService = this.grpcClient.getService<AuthServiceInterface>(
|
||||
ProtoServices.AuthProtoService,
|
||||
);
|
||||
|
||||
this.userService = this.grpcClient.getService<UsersProtoService>(
|
||||
ProtoServices.UsersProtoService,
|
||||
);
|
||||
}
|
||||
|
||||
async getPublicKeys() {
|
||||
@@ -53,25 +79,63 @@ export class AuthClientService implements OnModuleInit {
|
||||
return lastValueFrom(this.authService.AuthSnowflakeSignIn(input));
|
||||
}
|
||||
|
||||
checkDedicatedProxy({ customer }: AuthSignInResponse) {
|
||||
const DEDICATED_PROXY = process.env.DEDICATED_PROXY || '';
|
||||
this.logger.info(
|
||||
'SignIn - Setting customer ID for dedicated proxy: ' + DEDICATED_PROXY,
|
||||
);
|
||||
this.logger.info('Customer ID: ' + customer.id);
|
||||
|
||||
if (DEDICATED_PROXY !== '' && DEDICATED_PROXY !== customer.id) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
|
||||
// Bloquear o customer de acesso o maestro publico
|
||||
this.logger.info(
|
||||
'Check if customer have network policy: ' + customer.modules,
|
||||
);
|
||||
const hasNetworkPolicyModule = customer.modules.includes('network-policy');
|
||||
if (hasNetworkPolicyModule && DEDICATED_PROXY === '') {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
}
|
||||
|
||||
async signIn(
|
||||
{ username, password, totp }: AuthSignInRequest,
|
||||
metadata: Metadata,
|
||||
) {
|
||||
this.logger.info('SignIn');
|
||||
|
||||
return lastValueFrom(
|
||||
this.authService.AuthSignIn({ username, password, totp }, metadata),
|
||||
);
|
||||
let result: AuthSignInResponse;
|
||||
|
||||
try {
|
||||
result = await lastValueFrom(
|
||||
this.authService.AuthSignIn({ username, password, totp }, metadata),
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error('SignIn - Error during sign-in');
|
||||
this.logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (result.customer) {
|
||||
this.checkDedicatedProxy(result);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async refreshAccessToken(
|
||||
{ refreshToken }: AuthRefreshAccessTokenRequest,
|
||||
{ refreshToken, userId }: AuthRefreshAccessTokenRequest,
|
||||
metadata: Metadata,
|
||||
) {
|
||||
this.logger.info('RefreshAccessToken');
|
||||
|
||||
return lastValueFrom(
|
||||
this.authService.AuthRefreshAccessToken({ refreshToken }, metadata),
|
||||
this.authService.AuthRefreshAccessToken(
|
||||
{ refreshToken, userId },
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -79,6 +143,7 @@ export class AuthClientService implements OnModuleInit {
|
||||
accessToken,
|
||||
oldPassword,
|
||||
newPassword,
|
||||
totpCode,
|
||||
}: AuthChangePasswordRequest) {
|
||||
this.logger.info('ChangePassword');
|
||||
|
||||
@@ -87,6 +152,7 @@ export class AuthClientService implements OnModuleInit {
|
||||
accessToken,
|
||||
oldPassword,
|
||||
newPassword,
|
||||
totpCode,
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -239,4 +305,190 @@ export class AuthClientService implements OnModuleInit {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public async validateUserSession(accessToken: any, resourceHost: string) {
|
||||
const payload = await this.validateJwtToken(accessToken);
|
||||
|
||||
const userDto = await this.getUserfromPayload(payload);
|
||||
|
||||
this.validateResourceAccess(resourceHost, userDto);
|
||||
return userDto;
|
||||
}
|
||||
|
||||
public async refreshUserSession(
|
||||
refreshToken: string,
|
||||
userId: string,
|
||||
originHeader: string,
|
||||
): Promise<{
|
||||
user: UserDTO;
|
||||
authSession: AuthSession;
|
||||
}> {
|
||||
const metadata = PackTheMetadata({});
|
||||
|
||||
this.logger.info('Call Refresh Token');
|
||||
const refreshCredentials = await this.refreshAccessToken(
|
||||
{ refreshToken, userId },
|
||||
metadata,
|
||||
);
|
||||
this.logger.info('Finish Refresh Token');
|
||||
|
||||
const userDto = await this.validateUserSession(
|
||||
refreshCredentials.accessToken,
|
||||
originHeader,
|
||||
);
|
||||
return {
|
||||
user: userDto,
|
||||
authSession: {
|
||||
accessToken: refreshCredentials.accessToken,
|
||||
refreshToken: refreshCredentials.refreshToken,
|
||||
userId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
public writeAuthSession(res: Response, data: AuthSession) {
|
||||
let exp = 1000 * 60 * 5; // 5 minutes
|
||||
|
||||
if (data.accessToken) {
|
||||
const { exp: expiration } = jwt.decode(data.accessToken) as JwtPayload;
|
||||
exp = (expiration - 30) * 1000; // exp em segundos, maxAge em ms
|
||||
|
||||
this.logger.info('Set Cookie ddf-auth');
|
||||
res.cookie('ddf-auth', data.accessToken, {
|
||||
domain: '.dadosfera.ai',
|
||||
maxAge: exp,
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
sameSite: 'none', // Necessário para cookies em requisições cross-site
|
||||
});
|
||||
}
|
||||
|
||||
if (data.refreshToken) {
|
||||
this.logger.info('Set Cookie ddf-refresh-auth');
|
||||
res.cookie('ddf-refresh-auth', data.refreshToken, {
|
||||
domain: '.dadosfera.ai',
|
||||
maxAge: exp,
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
sameSite: 'none', // Necessário para cookies em requisições cross-site
|
||||
});
|
||||
}
|
||||
|
||||
if (data.userId) {
|
||||
this.logger.info('Set Cookie ddf-refresh-auth');
|
||||
res.cookie('ddf-user-id', data.userId, {
|
||||
domain: '.dadosfera.ai',
|
||||
maxAge: exp,
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
sameSite: 'none', // Necessário para cookies em requisições cross-site
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public cleanUpAuthSession(res: Response) {
|
||||
const exp = 1000 * 60 * 3;
|
||||
|
||||
res.cookie('ddf-auth', '', {
|
||||
domain: 'dadosfera.ai',
|
||||
maxAge: Date.now() - exp,
|
||||
expires: new Date(),
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
sameSite: 'none', // Necessário para cookies em requisições cross-site
|
||||
});
|
||||
|
||||
res.cookie('ddf-refresh-auth', '', {
|
||||
domain: 'dadosfera.ai',
|
||||
maxAge: Date.now() - exp,
|
||||
expires: new Date(),
|
||||
httpOnly: true,
|
||||
secure: true,
|
||||
sameSite: 'none', // Necessário para cookies em requisições cross-site
|
||||
});
|
||||
|
||||
this.logger.info('Clean cookie sessions');
|
||||
}
|
||||
|
||||
private async validateJwtToken(token: string) {
|
||||
const decoded: any = token && jwt.decode(token, { complete: true });
|
||||
if (!decoded) throw new Error('Invalid token');
|
||||
|
||||
const { kid } = decoded.header;
|
||||
// Busca a chave pública
|
||||
const { keys } = await this.getPublicKeys();
|
||||
const pemValue = keys.find((k) => k.kid === kid)?.pem;
|
||||
if (!pemValue) throw new Error('Public key not found');
|
||||
jwt.verify(token, pemValue);
|
||||
|
||||
return decoded.payload;
|
||||
}
|
||||
|
||||
private async getUserfromPayload(payload: JwtPayload): Promise<UserDTO> {
|
||||
this.logger.info('getUser');
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id: payload.customer_id,
|
||||
});
|
||||
|
||||
const { user } = await lastValueFrom(
|
||||
this.userService.UserFindOneById({ id: payload.user_id }, metadata),
|
||||
);
|
||||
|
||||
const userDto: UserDTO = {
|
||||
id: user.id,
|
||||
name: user.username,
|
||||
jobTitle: user?.jobTitle || null,
|
||||
department: user?.department || null,
|
||||
hierarchy: user?.hierarchy || null,
|
||||
customer: {
|
||||
id: payload.customer_id,
|
||||
name: payload.customer_name,
|
||||
tier: payload.customer_tier,
|
||||
},
|
||||
};
|
||||
|
||||
return userDto;
|
||||
}
|
||||
|
||||
private validateResourceAccess(host: string, user: UserDTO) {
|
||||
this.logger.info(
|
||||
"Validate whether the source URL is a resource belonging to the user's client",
|
||||
);
|
||||
this.logger.info('Host: ' + host);
|
||||
this.logger.info('Customer: ' + user.customer.name);
|
||||
|
||||
const hostParts = host.split('.');
|
||||
const domain = hostParts[0];
|
||||
const isResouceStg = hostParts[1] === 'stg';
|
||||
|
||||
const notFoundCustomerInDomain = !domain.includes('-')
|
||||
|
||||
if (notFoundCustomerInDomain) {
|
||||
this.logger.info(`Not found Customer Name in domain`);
|
||||
return;
|
||||
}
|
||||
|
||||
const domainParts = domain.split('-');
|
||||
|
||||
const customerInDomain = domainParts[domainParts.length - 1];
|
||||
|
||||
if (isResouceStg && process.env.ENV !== 'stg') {
|
||||
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
|
||||
throw new HttpException(
|
||||
`Customer ${user.customer.name} cannot access ${host}`,
|
||||
HttpStatus.FORBIDDEN
|
||||
);
|
||||
}
|
||||
|
||||
if (customerInDomain != user.customer.name) {
|
||||
this.logger.error(`Customer ${user.customer.name} cannot access ${host}`);
|
||||
throw new HttpException(
|
||||
`Customer ${user.customer.name} cannot access ${host}`,
|
||||
HttpStatus.FORBIDDEN
|
||||
);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ export class AuthRefreshAccessTokenReq {
|
||||
@ApiProperty()
|
||||
refreshToken: string;
|
||||
@ApiProperty()
|
||||
customerName: string;
|
||||
userId: string;
|
||||
}
|
||||
export class AuthRefreshAccessTokenRes {
|
||||
@ApiProperty()
|
||||
@@ -140,3 +140,16 @@ export interface BulkEditResponse {
|
||||
successfulUsers: string[];
|
||||
failedUsers: string[];
|
||||
}
|
||||
|
||||
export type UserDTO = {
|
||||
id: string,
|
||||
name: string,
|
||||
jobTitle?: string,
|
||||
department?: string,
|
||||
hierarchy?: string,
|
||||
customer: {
|
||||
id: string,
|
||||
name: string,
|
||||
tier: string,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ export class GoogleLoginStrategy extends PassportStrategy(
|
||||
callbackURL: oauthSecrets['google-login'].redirect_uri,
|
||||
scope: ['email', 'profile', 'openid'],
|
||||
};
|
||||
console.log("GoogleLoginStrategy", options.clientID, options.callbackURL);
|
||||
const verify = (
|
||||
accessToken: string,
|
||||
refreshToken: string,
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_ASSISTANT_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_ASSISTANT_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_ASSISTANT_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_ASSISTANT_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,201 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveAssistantService } from './autodrive-assistant.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Assistant')
|
||||
@Controller('autodrive-assistant')
|
||||
@Authenticated()
|
||||
export class AutodriveAssistantController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveAssistantService: AutodriveAssistantService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// ASSISTANTS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List assistants' })
|
||||
@Get('assistants')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async listAssistants(
|
||||
@Query('limit') limit: number,
|
||||
@Query('offset') offset: number,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
'/assistants',
|
||||
user,
|
||||
undefined,
|
||||
{ limit, offset },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create assistant' })
|
||||
@Post('assistants')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async createAssistant(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'POST',
|
||||
'/assistants',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get assistant by ID' })
|
||||
@Get('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async getAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update assistant' })
|
||||
@Put('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async updateAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'PUT',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete assistant' })
|
||||
@Delete('assistants/:assistantId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async deleteAssistant(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'DELETE',
|
||||
`/assistants/${assistantId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// KNOWLEDGE BASE
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List knowledge bases linked to assistant' })
|
||||
@Get('assistants/:assistantId/knowledge-bases')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async listKnowledgeBases(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/assistants/${assistantId}/knowledge-bases`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update knowledge base associations for assistant' })
|
||||
@Put('assistants/:assistantId/knowledge-bases')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async updateKnowledgeBases(
|
||||
@Param('assistantId') assistantId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'PUT',
|
||||
`/assistants/${assistantId}/knowledge-bases`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// QUESTIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Ask question via assistant' })
|
||||
@Post('dataset/:datasetId/ai_question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.WRITE)
|
||||
async aiQuestion(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/ai_question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get AI question answer' })
|
||||
@Get('dataset/:datasetId/ai_question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async getAiQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveAssistantService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/ai_question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UTILITY
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveAssistantService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_ASSISTANT.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveAssistantService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveAssistantController } from './autodrive-assistant.controller';
|
||||
import { AutodriveAssistantService } from './autodrive-assistant.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveAssistantController],
|
||||
providers: [AutodriveAssistantService, DadosferaLogger],
|
||||
exports: [AutodriveAssistantService],
|
||||
})
|
||||
export class AutodriveAssistantModule {}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_ASSISTANT_CONFIG } from './autodrive-assistant.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveAssistantService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive assistant operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_ASSISTANT_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-assistant', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_ASSISTANT_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Assistant API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Assistant API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Assistant API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_CORE_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_CORE_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_CORE_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_CORE_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,286 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
UseInterceptors,
|
||||
UploadedFiles,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
|
||||
import { FilesInterceptor } from '@nestjs/platform-express';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import FormData from 'form-data';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveCoreService } from './autodrive-core.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Core')
|
||||
@Controller('autodrive-core')
|
||||
@Authenticated()
|
||||
export class AutodriveCoreController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveCoreService: AutodriveCoreService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// DATASET MANAGEMENT
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'List datasets' })
|
||||
@Get('datasets')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async listDatasets(
|
||||
@Query('name') name: string,
|
||||
@Query('limit') limit: number,
|
||||
@Query('offset') offset: number,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
'/datasets',
|
||||
user,
|
||||
undefined,
|
||||
{ name, limit, offset },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get dataset by ID' })
|
||||
@Get('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Add documents to existing dataset' })
|
||||
@Put('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async updateDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'PUT',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete dataset' })
|
||||
@Delete('dataset/:datasetId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async deleteDataset(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'DELETE',
|
||||
`/dataset/${datasetId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Append documents from URLs to dataset' })
|
||||
@Post('dataset/:datasetId/append-from-url')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async appendFromUrl(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/append-from-url`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get dataset creation/update logs' })
|
||||
@Get('dataset/:datasetId/logs')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getDatasetLogs(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/logs`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// FILE OPERATIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Create vector dataset from uploaded files' })
|
||||
@Post('upload')
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@UseInterceptors(FilesInterceptor('files'))
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async upload(
|
||||
@UploadedFiles() files: Array<Express.Multer.File>,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const formData = new FormData();
|
||||
if (files && files.length > 0) {
|
||||
files.forEach((file) => {
|
||||
formData.append('files', file.buffer, {
|
||||
filename: file.originalname,
|
||||
contentType: file.mimetype,
|
||||
});
|
||||
});
|
||||
}
|
||||
// Forward additional body fields
|
||||
if (body) {
|
||||
Object.entries(body).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null && key !== 'files') {
|
||||
formData.append(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
return this.autodriveCoreService.proxyFormData(
|
||||
'POST',
|
||||
'/upload',
|
||||
user,
|
||||
formData,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create vector dataset from URLs' })
|
||||
@Post('upload-from-url')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.WRITE)
|
||||
async uploadFromUrl(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
'/upload-from-url',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// QUERY OPERATIONS
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Search dataset using semantic search' })
|
||||
@Post('dataset/:datasetId/question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async question(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get semantic search results' })
|
||||
@Get('dataset/:datasetId/question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Ask AI question on dataset' })
|
||||
@Post('dataset/:datasetId/ai_question')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async aiQuestion(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'POST',
|
||||
`/dataset/${datasetId}/ai_question`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get AI question answer' })
|
||||
@Get('dataset/:datasetId/ai_question/:questionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getAiQuestionResult(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('questionId') questionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveCoreService.proxy(
|
||||
'GET',
|
||||
`/dataset/${datasetId}/ai_question/${questionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// SYSTEM
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get frontend configuration' })
|
||||
@Get('config')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getConfig(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/config', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get usage metrics' })
|
||||
@Get('usage')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_CORE.permissions.READ)
|
||||
async getUsage(@User() user: RequestUser) {
|
||||
return this.autodriveCoreService.proxy('GET', '/usage', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveCoreController } from './autodrive-core.controller';
|
||||
import { AutodriveCoreService } from './autodrive-core.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveCoreController],
|
||||
providers: [AutodriveCoreService, DadosferaLogger],
|
||||
exports: [AutodriveCoreService],
|
||||
})
|
||||
export class AutodriveCoreModule {}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_CORE_CONFIG } from './autodrive-core.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveCoreService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive core operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_CORE_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-core', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_CORE_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Core API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Core API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Core API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
|
||||
async proxyFormData(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
formData: any,
|
||||
query?: Record<string, any>,
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive core operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_CORE_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
...formData.getHeaders?.(),
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying form data request to autodrive-core', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: formData,
|
||||
timeout: AUTODRIVE_CORE_CONFIG.timeout,
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Core API form data proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export const AUTODRIVE_EXTRACTOR_CONFIG = {
|
||||
getUrl: (customerName: string): string => {
|
||||
const urlTemplate = process.env.AUTODRIVE_EXTRACTOR_API_URL;
|
||||
if (!urlTemplate) {
|
||||
throw new Error('AUTODRIVE_EXTRACTOR_API_URL environment variable is not set');
|
||||
}
|
||||
return urlTemplate.replace('{customer}', customerName);
|
||||
},
|
||||
timeout: parseInt(process.env.AUTODRIVE_EXTRACTOR_TIMEOUT || '30000', 10),
|
||||
};
|
||||
@@ -0,0 +1,491 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Put,
|
||||
Delete,
|
||||
Param,
|
||||
Body,
|
||||
Query,
|
||||
Inject,
|
||||
UseInterceptors,
|
||||
UploadedFiles,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger';
|
||||
import { FilesInterceptor } from '@nestjs/platform-express';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import FormData from 'form-data';
|
||||
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { User, RequestUser } from '../../decorators/user.decorator';
|
||||
import { AutodriveExtractorService } from './autodrive-extractor.service';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
|
||||
@ApiTags('Autodrive Extractor')
|
||||
@Controller('autodrive-extractor')
|
||||
@Authenticated()
|
||||
export class AutodriveExtractorController {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
private readonly autodriveExtractorService: AutodriveExtractorService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LEGACY - NPL Extraction routes (to be removed)
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Extract features from dataset using NPL' })
|
||||
@Post('npl/datasets/:datasetId/extract')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplExtract(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
`/npl/datasets/${datasetId}/extract`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] List NPL extractions for a dataset' })
|
||||
@Get('npl/datasets/:datasetId/extractions')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplListExtractions(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Query('status') status: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions`,
|
||||
user,
|
||||
undefined,
|
||||
{ status },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL extraction details' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetExtraction(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL extraction to CSV' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportExtractionCsv(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL extraction to XLSX' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/xlsx')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportExtractionXlsx(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/xlsx`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Batch export all NPL extractions to CSV' })
|
||||
@Get('npl/datasets/:datasetId/extractions/export/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplBatchExportCsv(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/export/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Batch export all NPL extractions to XLSX' })
|
||||
@Get('npl/datasets/:datasetId/extractions/export/xlsx')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplBatchExportXlsx(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/export/xlsx`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL extraction report' })
|
||||
@Get('npl/datasets/:datasetId/extractions/:extractionId/report')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetExtractionReport(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Param('extractionId') extractionId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/datasets/${datasetId}/extractions/${extractionId}/report`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LEGACY - NPL Configuration routes (to be removed)
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration' })
|
||||
@Get('npl/config')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration summary' })
|
||||
@Get('npl/config/summary')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigSummary(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/summary', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] List NPL configuration fields' })
|
||||
@Get('npl/config/fields')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigFields(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/fields', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL configuration field by ID' })
|
||||
@Get('npl/config/fields/:fieldId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetConfigField(
|
||||
@Param('fieldId') fieldId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/npl/config/fields/${fieldId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Update NPL configuration field' })
|
||||
@Put('npl/config/fields/:fieldId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplUpdateConfigField(
|
||||
@Param('fieldId') fieldId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
`/npl/config/fields/${fieldId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL RAG queries' })
|
||||
@Get('npl/config/rag-queries')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetRagQueries(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/rag-queries', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL system prompt' })
|
||||
@Get('npl/config/system-prompt')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetSystemPrompt(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/system-prompt', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Update NPL system prompt' })
|
||||
@Put('npl/config/system-prompt')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplUpdateSystemPrompt(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
'/npl/config/system-prompt',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Get NPL source priorities' })
|
||||
@Get('npl/config/source-priorities')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplGetSourcePriorities(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/source-priorities', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Reset NPL configuration to defaults' })
|
||||
@Post('npl/config/reset')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplResetConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('POST', '/npl/config/reset', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Export NPL configuration' })
|
||||
@Get('npl/config/export')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async nplExportConfig(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/npl/config/export', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: '[LEGACY] Import NPL configuration' })
|
||||
@Post('npl/config/import')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async nplImportConfig(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/npl/config/import',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// EXTRACTION TEMPLATES
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Seed NPL Brasil extraction template' })
|
||||
@Post('templates/seed-npl')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async seedNplTemplate(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/templates/seed-npl',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'List extraction templates' })
|
||||
@Get('templates')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async listTemplates(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/templates', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Create extraction template' })
|
||||
@Post('templates')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async createTemplate(
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
'/templates',
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction template by ID' })
|
||||
@Get('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Update extraction template' })
|
||||
@Put('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async updateTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'PUT',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Delete extraction template' })
|
||||
@Delete('templates/:templateId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async deleteTemplate(
|
||||
@Param('templateId') templateId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'DELETE',
|
||||
`/templates/${templateId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Import extraction templates from file' })
|
||||
@Post('templates/import')
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@UseInterceptors(FilesInterceptor('file'))
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async importTemplates(
|
||||
@UploadedFiles() files: Array<Express.Multer.File>,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const formData = new FormData();
|
||||
if (files && files.length > 0) {
|
||||
files.forEach((file) => {
|
||||
formData.append('file', file.buffer, {
|
||||
filename: file.originalname,
|
||||
contentType: file.mimetype,
|
||||
});
|
||||
});
|
||||
}
|
||||
return this.autodriveExtractorService.proxyFormData(
|
||||
'POST',
|
||||
'/templates/import',
|
||||
user,
|
||||
formData,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// GENERIC EXTRACTION
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Start template-based extraction on dataset' })
|
||||
@Post('extraction/datasets/:datasetId/extract')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.WRITE)
|
||||
async startExtraction(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Body() body: any,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'POST',
|
||||
`/extraction/datasets/${datasetId}/extract`,
|
||||
user,
|
||||
body,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'List extraction jobs for a dataset' })
|
||||
@Get('extraction/datasets/:datasetId/jobs')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async listExtractionJobs(
|
||||
@Param('datasetId') datasetId: string,
|
||||
@Query('status') status: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/datasets/${datasetId}/jobs`,
|
||||
user,
|
||||
undefined,
|
||||
{ status },
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction job details' })
|
||||
@Get('extraction/jobs/:jobId')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getExtractionJob(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Get extraction job report' })
|
||||
@Get('extraction/jobs/:jobId/report')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async getExtractionJobReport(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}/report`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Export extraction job to CSV' })
|
||||
@Get('extraction/jobs/:jobId/csv')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async exportExtractionJobCsv(
|
||||
@Param('jobId') jobId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
return this.autodriveExtractorService.proxy(
|
||||
'GET',
|
||||
`/extraction/jobs/${jobId}/csv`,
|
||||
user,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// UTILITY
|
||||
// ============================================
|
||||
|
||||
@ApiOperation({ summary: 'Health check' })
|
||||
@Get('health')
|
||||
async health(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/health', user);
|
||||
}
|
||||
|
||||
@ApiOperation({ summary: 'Check model availability' })
|
||||
@Get('model-availability')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.AUTODRIVE_EXTRACTOR.permissions.READ)
|
||||
async modelAvailability(@User() user: RequestUser) {
|
||||
return this.autodriveExtractorService.proxy('GET', '/model-availability', user);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { AutodriveExtractorController } from './autodrive-extractor.controller';
|
||||
import { AutodriveExtractorService } from './autodrive-extractor.service';
|
||||
|
||||
@Module({
|
||||
imports: [],
|
||||
controllers: [AutodriveExtractorController],
|
||||
providers: [AutodriveExtractorService, DadosferaLogger],
|
||||
exports: [AutodriveExtractorService],
|
||||
})
|
||||
export class AutodriveExtractorModule {}
|
||||
@@ -0,0 +1,165 @@
|
||||
import { Injectable, Inject, HttpException } from '@nestjs/common';
|
||||
import axios, { AxiosResponse, Method } from 'axios';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { AUTODRIVE_EXTRACTOR_CONFIG } from './autodrive-extractor.config';
|
||||
|
||||
@Injectable()
|
||||
export class AutodriveExtractorService {
|
||||
private logger: any;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async proxy(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
body?: any,
|
||||
query?: Record<string, any>
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive extractor operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_EXTRACTOR_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying request to autodrive-extractor', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: body,
|
||||
timeout: AUTODRIVE_EXTRACTOR_CONFIG.timeout,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Extractor API proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
if (error.code === 'ECONNREFUSED') {
|
||||
throw new HttpException('Autodrive Extractor API service unavailable', 503);
|
||||
}
|
||||
|
||||
if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') {
|
||||
throw new HttpException('Autodrive Extractor API request timeout', 504);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
|
||||
async proxyFormData(
|
||||
method: string,
|
||||
path: string,
|
||||
user: RequestUser,
|
||||
formData: any,
|
||||
query?: Record<string, any>,
|
||||
): Promise<any> {
|
||||
if (!user.customer_id) {
|
||||
throw new HttpException('Customer ID is required for autodrive extractor operations', 400);
|
||||
}
|
||||
|
||||
const baseUrl = AUTODRIVE_EXTRACTOR_CONFIG.getUrl(user.customer_name);
|
||||
const url = new URL(`${baseUrl}${path}`);
|
||||
|
||||
if (query) {
|
||||
Object.entries(query).forEach(([key, value]) => {
|
||||
if (value !== undefined && value !== null) {
|
||||
url.searchParams.set(key, String(value));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
...formData.getHeaders?.(),
|
||||
'Authorization': user.access_token,
|
||||
};
|
||||
|
||||
this.logger.info('Proxying form data request to autodrive-extractor', {
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
});
|
||||
|
||||
try {
|
||||
const response: AxiosResponse = await axios({
|
||||
method: method as Method,
|
||||
url: url.href,
|
||||
headers,
|
||||
data: formData,
|
||||
timeout: AUTODRIVE_EXTRACTOR_CONFIG.timeout,
|
||||
maxContentLength: Infinity,
|
||||
maxBodyLength: Infinity,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
|
||||
if (response.status >= 400) {
|
||||
throw new HttpException(response.data, response.status);
|
||||
}
|
||||
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
this.logger.error('Autodrive Extractor API form data proxy error', {
|
||||
error: error.message,
|
||||
status: error.response?.status,
|
||||
path,
|
||||
method: method.toUpperCase(),
|
||||
});
|
||||
|
||||
if (error instanceof HttpException) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (error.response) {
|
||||
throw new HttpException(error.response.data, error.response.status);
|
||||
}
|
||||
|
||||
throw new HttpException('Internal server error', 500);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
UseFilters,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Res,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiCreatedResponse,
|
||||
@@ -25,14 +26,14 @@ import {
|
||||
import {
|
||||
Authenticated,
|
||||
RequireAllPermissions,
|
||||
RequireModule,
|
||||
RequireSomePermission,
|
||||
} from '../../decorators/authentication.decorator';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { CatalogService } from './catalog.service';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import {
|
||||
BatchRemoveRlsRulesRequest,
|
||||
GetDatasetCatalogTaskRes,
|
||||
@@ -59,6 +60,8 @@ import {
|
||||
GetRlsRulesRequest,
|
||||
RegisterDatasetWithMetatadaRequest,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||
import { Response } from 'express';
|
||||
import { TypeParser } from 'src/utils/FileParser/parser-types';
|
||||
|
||||
@ApiTags('Catalog')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@@ -114,6 +117,52 @@ export class CatalogController {
|
||||
return res;
|
||||
}
|
||||
|
||||
@Get('/download')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async dowloadAsserts(
|
||||
@User() user: RequestUser,
|
||||
@Query() query: ICatalogAllRequest,
|
||||
@Res() res: Response
|
||||
) {
|
||||
const { user_id, customer_name, customer_id, username, permissions } = user;
|
||||
this.logger.info(`/catalog/download - searchCatalog`, {
|
||||
user_id,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
const is_data_manager = permissions.includes(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
|
||||
);
|
||||
|
||||
const roles = await this.catalogService.getUserRolesIds(user_id);
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
user_id,
|
||||
customer_id,
|
||||
customer_name,
|
||||
username,
|
||||
roles,
|
||||
is_data_manager,
|
||||
});
|
||||
|
||||
const {
|
||||
file,
|
||||
filename
|
||||
} = await this.catalogService.downloadAssets(
|
||||
query,
|
||||
metadata,
|
||||
customer_id,
|
||||
);
|
||||
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
|
||||
res.setHeader('Content-Type', 'text/csv');
|
||||
|
||||
res.end(file);
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Get('data-asset')
|
||||
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
|
||||
@@ -344,7 +393,7 @@ export class CatalogController {
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
): Promise<IPreviewResponse> {
|
||||
const { customer_name, customer_id, user_id, username } = user;
|
||||
const { customer_name, customer_id, user_id, username, customer_modules } = user;
|
||||
|
||||
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
|
||||
user_id,
|
||||
@@ -357,6 +406,7 @@ export class CatalogController {
|
||||
user_id,
|
||||
username,
|
||||
language,
|
||||
is_mask: customer_modules.some(mod => mod === 'pii')
|
||||
});
|
||||
|
||||
const preview = await this.catalogService.getDatasetPreview(id, metadata);
|
||||
@@ -373,6 +423,7 @@ export class CatalogController {
|
||||
@User() user: RequestUser,
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@Query('asset_type') asset_type: string,
|
||||
): Promise<IDocsResponse> {
|
||||
const { customer_name, customer_id, user_id, username } = user;
|
||||
|
||||
@@ -389,7 +440,7 @@ export class CatalogController {
|
||||
language,
|
||||
});
|
||||
|
||||
const docs = await this.catalogService.getDataDocs(id, metadata);
|
||||
const docs = await this.catalogService.getDataDocs(id, asset_type, metadata);
|
||||
|
||||
return { docs };
|
||||
}
|
||||
@@ -437,21 +488,32 @@ export class CatalogController {
|
||||
@Headers() headers,
|
||||
@Param('id') table_id: string,
|
||||
@Body('docs') docs: string,
|
||||
@Query('asset_type') asset_type: string,
|
||||
) {
|
||||
const { user_id, customer_name } = user;
|
||||
const { user_id, customer_name, customer_id, username } = user;
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
});
|
||||
|
||||
this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, {
|
||||
this.logger.info(`/catalog - ON POST DATA DOCS ROUTE`, {
|
||||
user_id,
|
||||
customer_name,
|
||||
});
|
||||
|
||||
const res = await this.catalogService.createDataDocs({
|
||||
const body = {
|
||||
table_id,
|
||||
docs,
|
||||
asset_type,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const res = await this.catalogService.createDataDocs(body, metadata);
|
||||
|
||||
return res;
|
||||
}
|
||||
@@ -748,25 +810,23 @@ export class CatalogController {
|
||||
return JSON.parse(dashboards);
|
||||
}
|
||||
|
||||
|
||||
@Post('register-dataset')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
||||
)
|
||||
async registerDatasetWithMetadataRequest(
|
||||
@Body() body: RegisterDatasetWithMetatadaRequest,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const logMetadata = {
|
||||
customer_name: customer_name,
|
||||
user_id: user_id,
|
||||
method: 'POST',
|
||||
path: '/catalog/register-dataset',
|
||||
};
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const logMetadata = {
|
||||
customer_name: customer_name,
|
||||
user_id: user_id,
|
||||
method: 'POST',
|
||||
path: '/catalog/register-dataset',
|
||||
};
|
||||
try {
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
@@ -774,18 +834,20 @@ export class CatalogController {
|
||||
username,
|
||||
});
|
||||
|
||||
this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata);
|
||||
this.logger.log(
|
||||
`Request from user ${user_id} for customer ${customer_name}`,
|
||||
logMetadata,
|
||||
);
|
||||
|
||||
// Create table metadata
|
||||
const tableMetadataBody = {
|
||||
table_metadata: body.table_metadata,
|
||||
info: {
|
||||
customer: customer_name,
|
||||
|
||||
},
|
||||
logMetadata: logMetadata
|
||||
logMetadata: logMetadata,
|
||||
};
|
||||
|
||||
|
||||
const table_metadata_id = await this.catalogService.createTableMetadata(
|
||||
tableMetadataBody,
|
||||
);
|
||||
@@ -798,8 +860,12 @@ export class CatalogController {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata);
|
||||
const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody)
|
||||
this.logger.info(
|
||||
`Creating column metadata for table ${table_metadata_id}`,
|
||||
logMetadata,
|
||||
);
|
||||
const column_metadata_ids =
|
||||
await this.catalogService.createColumnMetadata(columnMetadataBody);
|
||||
|
||||
// Create data preview
|
||||
const dataPreviewBody = {
|
||||
@@ -808,15 +874,24 @@ export class CatalogController {
|
||||
customer: customer_name,
|
||||
},
|
||||
};
|
||||
this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata);
|
||||
this.logger.debug(
|
||||
`Creating data preview for table ${table_metadata_id}`,
|
||||
logMetadata,
|
||||
);
|
||||
const data_preview_id = await this.catalogService.createDataPreview(
|
||||
dataPreviewBody,
|
||||
);
|
||||
|
||||
// Catalog dataset item
|
||||
this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata);
|
||||
this.logger.info(
|
||||
`Cataloging dataset item for table ${table_metadata_id}`,
|
||||
logMetadata,
|
||||
);
|
||||
await this.catalogService.catalogDatasetItem(table_metadata_id, metadata);
|
||||
this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata);
|
||||
this.logger.info(
|
||||
`Dataset registration completed successfully for table ${table_metadata_id}`,
|
||||
logMetadata,
|
||||
);
|
||||
return {
|
||||
message: 'Dataset registered successfully',
|
||||
table_metadata_id: table_metadata_id,
|
||||
@@ -824,7 +899,10 @@ export class CatalogController {
|
||||
data_preview_id: data_preview_id,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata);
|
||||
this.logger.error(
|
||||
`Failed to register dataset. The following error occurred: ${error.response.data}`,
|
||||
logMetadata,
|
||||
);
|
||||
|
||||
throw new HttpException(
|
||||
{
|
||||
@@ -837,4 +915,32 @@ export class CatalogController {
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Get('pii-reporter')
|
||||
@RequireSomePermission(
|
||||
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
|
||||
)
|
||||
@RequireModule(
|
||||
DADOSFERA_MODULES_KEYS.PII
|
||||
)
|
||||
async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") {
|
||||
this.logger.info('GET pii-reporter');
|
||||
const metadata = PackTheMetadata(user);
|
||||
|
||||
try {
|
||||
const {
|
||||
file,
|
||||
filename,
|
||||
type
|
||||
} = await this.catalogService.getPiiReporter(metadata, contentType);
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
|
||||
res.setHeader('Content-Type', type);
|
||||
|
||||
// use res.end to send buffer
|
||||
return res.end(file);
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
this.logger.error(error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,12 +3,15 @@ import { Module } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { CatalogController } from './catalog.controller';
|
||||
import { CatalogService } from './catalog.service';
|
||||
import { CatalogClientConfiguration } from './catalog-client';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
|
||||
import { UsersModule } from '../users/users.module';
|
||||
import { RolesModule } from '../roles/roles.module';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
import { ShareModule } from './share/share.module';
|
||||
import { CatalogService } from './catalog.service';
|
||||
import { MixpanelModule } from '../mixpanel/mixpanel.module';
|
||||
|
||||
const client = new CatalogClientConfiguration();
|
||||
|
||||
@@ -18,6 +21,8 @@ const client = new CatalogClientConfiguration();
|
||||
OldPipelineModule,
|
||||
UsersModule,
|
||||
RolesModule,
|
||||
CustomersModule,
|
||||
ShareModule,
|
||||
],
|
||||
controllers: [CatalogController],
|
||||
providers: [CatalogService, DadosferaLogger],
|
||||
|
||||
@@ -24,8 +24,11 @@ import { CatalogClientConfiguration } from './catalog-client';
|
||||
import { UsersService } from '../users/users.service';
|
||||
import { RolesService } from '../roles/roles.service';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import {
|
||||
AssetReporter,
|
||||
BatchRemoveRlsRulesRequest,
|
||||
CreateDataDocsDTO,
|
||||
IUpdateDataRequest,
|
||||
TriggerCatalogReq,
|
||||
} from './dtos';
|
||||
@@ -33,7 +36,10 @@ import {
|
||||
AddRlsRuleRequest,
|
||||
GetNimbusDashboardsRequest,
|
||||
GetRlsRulesRequest,
|
||||
PiiMetadata,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||
import { TypeParser } from 'src/utils/FileParser/parser-types';
|
||||
import { ParserBuilder } from 'src/utils/FileParser/parser.builder';
|
||||
|
||||
class CatalogService implements OnModuleInit {
|
||||
catalogReadService: ReadService.CatalogReadServices;
|
||||
@@ -80,6 +86,39 @@ class CatalogService implements OnModuleInit {
|
||||
)}.dadosfera.ai`;
|
||||
}
|
||||
|
||||
async getPiiReporter(metadata: Metadata, type: TypeParser) {
|
||||
this.logger.info('getPiiReporter: ' + type);
|
||||
try {
|
||||
const { data } = await lastValueFrom(
|
||||
this.catalogWriteService.GetPiiReporter({}, metadata),
|
||||
);
|
||||
this.logger.info('Finish grpc call');
|
||||
|
||||
const parser = ParserBuilder.build<PiiMetadata>(type);
|
||||
|
||||
this.logger.info('parser file to: ' + type);
|
||||
const file = await parser.parse(data);
|
||||
this.logger.info('finish parser');
|
||||
const mimeTypes: Record<TypeParser, string> = {
|
||||
csv: 'text/csv',
|
||||
html: 'text/html',
|
||||
pdf: 'application/pdf',
|
||||
};
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const filename = `relatorio-pii-${timestamp}.${type}`;
|
||||
|
||||
return {
|
||||
file,
|
||||
filename: filename,
|
||||
type: mimeTypes[type],
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) {
|
||||
this.logger.info('CatalogService - Manage Data assets permissions');
|
||||
if (!data.embed) data.embed = undefined;
|
||||
@@ -157,9 +196,11 @@ class CatalogService implements OnModuleInit {
|
||||
async getUserRolesIds(userId: string) {
|
||||
const result = await this.userService.findOneById(userId).catch(() => null);
|
||||
|
||||
const roles_ids = result.user.roles.map((role) => role.id);
|
||||
if (result) {
|
||||
return result.user.roles.map((role) => role.id);
|
||||
}
|
||||
|
||||
return roles_ids;
|
||||
return [];
|
||||
}
|
||||
|
||||
async searchDataAssets(
|
||||
@@ -167,10 +208,99 @@ class CatalogService implements OnModuleInit {
|
||||
metadata: Metadata,
|
||||
customer_id: string,
|
||||
) {
|
||||
this.logger.info('CatalogService - searchDataAssets');
|
||||
this.logger.info('CatalogService - searchDataAssets', { query });
|
||||
|
||||
const { search, page, size, sort_by, order, ...filters } = query;
|
||||
|
||||
this.logger.debug('Extracted filters:', { filters });
|
||||
|
||||
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (ANTES AJUSTE):', {
|
||||
search,
|
||||
page,
|
||||
size,
|
||||
sort_by,
|
||||
order,
|
||||
filters,
|
||||
});
|
||||
|
||||
if (
|
||||
filters.manually !== undefined &&
|
||||
filters.manually !== null &&
|
||||
filters.manually !== ''
|
||||
) {
|
||||
filters.manually = Number(filters.manually); // 1 ou 0
|
||||
} else {
|
||||
delete filters.manually;
|
||||
}
|
||||
|
||||
console.log('MAESTRO VAI CHAMAR PI-FACTORY COM (DEPOIS AJUSTE):', {
|
||||
search,
|
||||
page,
|
||||
size,
|
||||
sort_by,
|
||||
order,
|
||||
filters,
|
||||
});
|
||||
|
||||
if (filters.owner) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
|
||||
this.logger.info('Available users in database count:', {
|
||||
count: customer_users.length,
|
||||
});
|
||||
this.logger.info('First 5 users:', {
|
||||
users: customer_users
|
||||
.slice(0, 5)
|
||||
.map((u) => ({ id: u.id, email: u.email, name: u.name })),
|
||||
});
|
||||
|
||||
const ownerValues = Array.isArray(filters.owner)
|
||||
? filters.owner
|
||||
: typeof filters.owner === 'string' && filters.owner.includes(',')
|
||||
? filters.owner.split(',').map((o: string) => o.trim())
|
||||
: [filters.owner];
|
||||
|
||||
this.logger.info('Owner values to convert:', {
|
||||
ownerValues,
|
||||
ownerFiltersOriginal: filters.owner,
|
||||
});
|
||||
|
||||
const ownerIds = ownerValues
|
||||
.map((ownerValue: string) => {
|
||||
const normalizedOwner = ownerValue.replace(/\s/g, '+');
|
||||
const user = customer_users.find((u) => {
|
||||
const isIdMatch = u.id === ownerValue;
|
||||
const isEmailMatch =
|
||||
u.email === ownerValue || u.email === normalizedOwner;
|
||||
const isNameMatch =
|
||||
u.name === ownerValue || u.name === normalizedOwner;
|
||||
this.logger.info('Comparing:', {
|
||||
userId: u.id,
|
||||
userEmail: u.email,
|
||||
userName: u.name,
|
||||
filterValue: ownerValue,
|
||||
normalizedFilter: normalizedOwner,
|
||||
idMatch: isIdMatch,
|
||||
emailMatch: isEmailMatch,
|
||||
nameMatch: isNameMatch,
|
||||
});
|
||||
return isIdMatch || isEmailMatch || isNameMatch;
|
||||
});
|
||||
this.logger.info('Looking for owner result:', {
|
||||
ownerValue,
|
||||
found: !!user,
|
||||
userId: user?.id,
|
||||
});
|
||||
return user?.id || ownerValue;
|
||||
})
|
||||
.filter((id: string) => id);
|
||||
|
||||
if (ownerIds.length > 0) {
|
||||
filters.owner = ownerIds;
|
||||
}
|
||||
}
|
||||
|
||||
const { data_assets, total } = await lastValueFrom(
|
||||
this.catalogReadService.GetAllDataAssets(
|
||||
{
|
||||
@@ -185,6 +315,8 @@ class CatalogService implements OnModuleInit {
|
||||
),
|
||||
);
|
||||
|
||||
console.log('MAESTRO RECEBEU RESPOSTA DO PI-FACTORY');
|
||||
|
||||
const result = JSON.parse(data_assets);
|
||||
|
||||
const response = await this.getAssetsUsersAndRoles(
|
||||
@@ -195,6 +327,34 @@ class CatalogService implements OnModuleInit {
|
||||
return { data_assets: response, total };
|
||||
}
|
||||
|
||||
async downloadAssets(
|
||||
query: Record<string, any>,
|
||||
metadata: Metadata,
|
||||
customer_id: string,
|
||||
) {
|
||||
const data = await this.searchDataAssets(query, metadata, customer_id);
|
||||
|
||||
const formatData = data.data_assets.map((asset) => ({
|
||||
id: asset.id,
|
||||
display_name: asset.display_name,
|
||||
data_asset_type: asset.data_asset_type,
|
||||
created_at: asset.created_at,
|
||||
tags: '[' + asset.tags.join(', ') + ']',
|
||||
}));
|
||||
|
||||
const parser = ParserBuilder.build<AssetReporter>('csv');
|
||||
|
||||
const file = await parser.parse(formatData);
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const filename = `dadosfera_assets_${timestamp}.csv`;
|
||||
|
||||
return {
|
||||
file,
|
||||
filename,
|
||||
};
|
||||
}
|
||||
|
||||
async getOneDataAsset(data: {
|
||||
id: string;
|
||||
customer_id: string;
|
||||
@@ -267,11 +427,11 @@ class CatalogService implements OnModuleInit {
|
||||
return { data_asset: asset[0] };
|
||||
}
|
||||
|
||||
async getDataDocs(id: string, metadata: Metadata) {
|
||||
async getDataDocs(id: string, assetType: string, metadata: Metadata) {
|
||||
const { documentation } = await lastValueFrom(
|
||||
this.catalogReadService.GetDatasetDoc({ id, type: undefined }, metadata),
|
||||
this.catalogReadService.GetDatasetDoc({ id }, metadata),
|
||||
);
|
||||
console.log(documentation);
|
||||
|
||||
const docs = JSON.parse(documentation);
|
||||
return docs;
|
||||
}
|
||||
@@ -298,7 +458,16 @@ class CatalogService implements OnModuleInit {
|
||||
return result;
|
||||
}
|
||||
|
||||
async createDataDocs(body) {
|
||||
async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
|
||||
if (body.asset_type === 'table' || body.asset_type === 'view') {
|
||||
return this.createDataDocsViaNimbus(body);
|
||||
}
|
||||
|
||||
return this.createDataDocsViaGrpc(body, metadata);
|
||||
}
|
||||
|
||||
private async createDataDocsViaNimbus(body: CreateDataDocsDTO) {
|
||||
this.logger.info('Creating data docs via Nimbus for table/view');
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
const { data } = await axios.post(
|
||||
`${nimbusUrl}/api/catalog/data-docs/`,
|
||||
@@ -307,6 +476,29 @@ class CatalogService implements OnModuleInit {
|
||||
return data;
|
||||
}
|
||||
|
||||
private async createDataDocsViaGrpc(body: CreateDataDocsDTO, metadata: Metadata) {
|
||||
this.logger.info('Creating data docs via gRPC for other asset types');
|
||||
try {
|
||||
const response: any = await lastValueFrom(
|
||||
this.catalogWriteService.UpdateDataAssetDoc(
|
||||
{
|
||||
id: body.table_id,
|
||||
docs: body.docs,
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
|
||||
return response;
|
||||
} catch (error) {
|
||||
this.logger.error('Error creating data asset docs:', error);
|
||||
throw new HttpException(
|
||||
'Failed to create data asset documentation',
|
||||
HttpStatus.INTERNAL_SERVER_ERROR,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async findAllTags(data, metadata) {
|
||||
this.logger.info('CatalogService - findAllCustomerTags');
|
||||
|
||||
@@ -324,6 +516,7 @@ class CatalogService implements OnModuleInit {
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
async getAssetsUsersAndRoles(data_assets: Array<any>, customer_id: string) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
@@ -334,17 +527,20 @@ class CatalogService implements OnModuleInit {
|
||||
return data_assets.map((data_asset) => {
|
||||
const owner = customer_users.find(
|
||||
(u) => u.id === data_asset.owner,
|
||||
)?.username;
|
||||
)?.email;
|
||||
|
||||
const roles = [];
|
||||
const users = [];
|
||||
for (const role_id of data_asset.roles) {
|
||||
const data_asset_roles = data_asset?.roles || [];
|
||||
for (const role_id of data_asset_roles) {
|
||||
const role = customer_roles.find((r) => r.id === role_id);
|
||||
if (role) roles.push({ id: role.id, name: role.name });
|
||||
}
|
||||
for (const user_id of data_asset.users) {
|
||||
|
||||
const data_asset_users = data_asset?.users || [];
|
||||
for (const user_id of data_asset_users) {
|
||||
const user = customer_users.find((r) => r.id === user_id);
|
||||
if (user) users.push({ id: user.id, username: user.username });
|
||||
if (user) users.push({ id: user.id, email: user.email });
|
||||
}
|
||||
return {
|
||||
...data_asset,
|
||||
@@ -434,27 +630,36 @@ class CatalogService implements OnModuleInit {
|
||||
|
||||
async createTableMetadata(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
|
||||
|
||||
const endpoint = `${nimbusUrl}/api/catalog/table-metadata/`;
|
||||
|
||||
this.logger.info(`Creating table metadata for table ${body.table_metadata.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {...body.logMetadata});
|
||||
this.logger.info(
|
||||
`Creating table metadata for table ${body.table_metadata.table_name}`,
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.table_metadata)}`, {
|
||||
...body.logMetadata,
|
||||
});
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, {...body.table_metadata});
|
||||
const { data, status } = await axios.post(endpoint, {
|
||||
...body.table_metadata,
|
||||
});
|
||||
|
||||
this.logger.info(
|
||||
`Table metadata created successfully with status ${status} for table ${body.table_metadata.table_name}`,
|
||||
{...body.logMetadata},
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
return data.id;
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create table metadata for table ${body.table_metadata.table_name} failed with status ${
|
||||
error.response?.status
|
||||
} because of ${JSON.stringify(error.response?.data) || error.message}`, {...body.logMetadata});
|
||||
} because of ${JSON.stringify(error.response?.data) || error.message}`,
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
@@ -464,43 +669,58 @@ class CatalogService implements OnModuleInit {
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, body.logMetadata);
|
||||
const endpoint = `${nimbusUrl}/api/catalog/column-metadata/`;
|
||||
|
||||
|
||||
|
||||
try {
|
||||
this.logger.info(`Creating column metadata for table ${body.column_metadata.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.column_metadata)}`, {...body.logMetadata});
|
||||
const { data, status } = await axios.post(endpoint, body.column_metadata);
|
||||
this.logger.info(
|
||||
`Creating column metadata for table ${body.column_metadata.table_name}`,
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
|
||||
this.logger.debug(
|
||||
`Payload: ${JSON.stringify(body.column_metadata)}`,
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
const { data, status } = await axios.post(
|
||||
endpoint,
|
||||
body.column_metadata,
|
||||
);
|
||||
|
||||
this.logger.info(
|
||||
`Column metadata created successfully with status ${status} for table ${body.column_metadata.table_name}`,
|
||||
{...body.logMetadata},
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
return data.map((column) => column.id);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`Failed to create column metadata failed with status for table ${body.column_metadata.table_name} ${
|
||||
error.response?.status
|
||||
} because of ${error.response?.data || error.message}`, {...body.logMetadata});
|
||||
} because of ${error.response?.data || error.message}`,
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async createDataPreview(body: any): Promise<number> {
|
||||
const nimbusUrl = this._getNimbusUrl(body);
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, {...body.logMetadata});
|
||||
this.logger.info(`Nimbus URL: ${nimbusUrl}`, { ...body.logMetadata });
|
||||
const endpoint = `${nimbusUrl}/api/catalog/data-preview/`;
|
||||
|
||||
this.logger.info(`Creating data preview for table ${body.data_preview.table_name}`, {...body.logMetadata});
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, {...body.logMetadata});
|
||||
this.logger.debug(`Payload: ${JSON.stringify(body.data_preview)}`, {...body.logMetadata});
|
||||
this.logger.info(
|
||||
`Creating data preview for table ${body.data_preview.table_name}`,
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
this.logger.info(`Using endpoint: ${endpoint}`, { ...body.logMetadata });
|
||||
this.logger.debug(
|
||||
`Payload: ${JSON.stringify(body.data_preview)}`,
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
|
||||
try {
|
||||
const { data, status } = await axios.post(endpoint, body.data_preview);
|
||||
|
||||
this.logger.info(
|
||||
`Data preview created successfully with status ${status} for table ${body.data_preview.table_name}`,
|
||||
{...body.logMetadata},
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
return data.id;
|
||||
} catch (error) {
|
||||
@@ -508,7 +728,7 @@ class CatalogService implements OnModuleInit {
|
||||
`Failed to create data preview for table ${body.data_preview.table_name} failed with status ${
|
||||
error.response?.status
|
||||
} because of ${error.response?.data || error.message}`,
|
||||
{...body.logMetadata},
|
||||
{ ...body.logMetadata },
|
||||
);
|
||||
throw new Error(error.response?.data?.message || error.message);
|
||||
}
|
||||
|
||||
@@ -98,6 +98,8 @@ export class IDataAsset {
|
||||
embed?: EmbedObject;
|
||||
@ApiPropertyOptional({ enum: DataAssetShareType })
|
||||
share_type?: DataAssetShareType;
|
||||
@ApiPropertyOptional()
|
||||
docs?: string;
|
||||
}
|
||||
|
||||
export class IOneDataAsset {
|
||||
@@ -147,6 +149,24 @@ export class ICatalogAllRequest {
|
||||
description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
|
||||
})
|
||||
order?: OrderEnum;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'ID do usuário owner para filtrar data assets',
|
||||
example: 'user-id-1,user-id-2',
|
||||
})
|
||||
owner?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Data inicial para filtro de catálogo (formato: YYYY-MM-DD)',
|
||||
example: '2025-01-01',
|
||||
})
|
||||
catalog_date_from?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Data final para filtro de catálogo (formato: YYYY-MM-DD)',
|
||||
example: '2025-12-31',
|
||||
})
|
||||
catalog_date_to?: string;
|
||||
}
|
||||
|
||||
export class ICatalogAllResponse {
|
||||
@@ -182,6 +202,8 @@ export class IUpdateDataRequest {
|
||||
embed: EmbedObject;
|
||||
@ApiPropertyOptional({ enum: DataAssetShareType })
|
||||
share_type?: DataAssetShareType;
|
||||
@ApiPropertyOptional()
|
||||
docs?: string;
|
||||
}
|
||||
export class ICreateDataAsset implements CreateDataAssetRequest {
|
||||
@ApiProperty()
|
||||
@@ -196,6 +218,8 @@ export class ICreateDataAsset implements CreateDataAssetRequest {
|
||||
location: string;
|
||||
@ApiPropertyOptional()
|
||||
embed: EmbedObject;
|
||||
@ApiPropertyOptional()
|
||||
docs: string;
|
||||
}
|
||||
|
||||
export class IPreview {
|
||||
@@ -320,3 +344,18 @@ export class BatchRemoveRlsRulesRequest {
|
||||
@ApiPropertyOptional()
|
||||
id_rls?: string;
|
||||
}
|
||||
|
||||
export type AssetReporter = {
|
||||
id: string;
|
||||
display_name: string;
|
||||
data_asset_type: string;
|
||||
created_at: string;
|
||||
tags: string;
|
||||
}
|
||||
|
||||
export type CreateDataDocsDTO = {
|
||||
table_id: string;
|
||||
docs: string;
|
||||
asset_type: string;
|
||||
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export class PiiDto {
|
||||
database_name: string;
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
column_name: string;
|
||||
data_type: string;
|
||||
pii_rules: string;
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Inject,
|
||||
Param,
|
||||
Req,
|
||||
UseFilters,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiHeaders,
|
||||
ApiTags,
|
||||
} from '@nestjs/swagger';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import {
|
||||
IColumnsMetadataResponse,
|
||||
IDocsResponse,
|
||||
IPreviewResponse,
|
||||
|
||||
} from '../dtos';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { ShareService } from './share.service';
|
||||
import { Request } from 'express';
|
||||
|
||||
@ApiTags('Catalog')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@Controller('catalog/data-asset/share')
|
||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||
export class ShareController {
|
||||
logger: DadosferaLogger;
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private catalogShareService: ShareService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Get('/:id')
|
||||
async getShareDataAsset(
|
||||
@Param('id') id: string,
|
||||
@Req() request: Request
|
||||
) {
|
||||
this.logger.info(`GET //:id`);
|
||||
return await this.catalogShareService.getOneDataAssetPublic(id, request);
|
||||
}
|
||||
|
||||
@Get('/:id/columns-metadata')
|
||||
async getShareDataAssetColumnsMetadata(
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@Req() request: Request
|
||||
): Promise<IColumnsMetadataResponse> {
|
||||
this.logger.info(`GET /:id/columns-metadata`);
|
||||
|
||||
const columns_metadata =
|
||||
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
|
||||
|
||||
|
||||
return { columns_metadata };
|
||||
}
|
||||
|
||||
@Get('/:id/preview')
|
||||
async getShareDataAssetPreview(
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@Req() request: Request
|
||||
): Promise<IPreviewResponse> {
|
||||
this.logger.info(`GET /:id/preview`);
|
||||
const preview = await this.catalogShareService.getDatasetPreview(id, request);
|
||||
|
||||
return { preview };
|
||||
}
|
||||
|
||||
@Get('/:id/docs')
|
||||
async getShareDataAssetDocs(
|
||||
@Language() language: LanguageEnum,
|
||||
@Param('id') id: string,
|
||||
@Req() request: Request
|
||||
): Promise<IDocsResponse> {
|
||||
this.logger.info(`GET /:id/docs`);
|
||||
const docs = await this.catalogShareService.getDataDocs(id, request);
|
||||
|
||||
return { docs };
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { CatalogClientConfiguration } from "../catalog-client";
|
||||
import { ClientsModule } from "@nestjs/microservices";
|
||||
import { RolesModule } from "src/modules/roles/roles.module";
|
||||
import { UsersModule } from "src/modules/users/users.module";
|
||||
import { CustomersModule } from "src/modules/customers/customers.module";
|
||||
import { ShareMetadataModule } from "src/modules/share-metadata/share-metadata.module";
|
||||
import { ShareController } from "./share.controller";
|
||||
import DadosferaLogger from "@dadosfera/dadosfera-logs";
|
||||
import { ShareService } from "./share.service";
|
||||
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
|
||||
import { AuthModule } from "src/modules/auth/auth.module";
|
||||
|
||||
const client = new CatalogClientConfiguration();
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ClientsModule.register([client.providerOptions]),
|
||||
UsersModule,
|
||||
RolesModule,
|
||||
CustomersModule,
|
||||
ShareMetadataModule,
|
||||
MixpanelModule,
|
||||
AuthModule
|
||||
],
|
||||
controllers: [ShareController],
|
||||
providers: [ShareService, DadosferaLogger],
|
||||
exports: [ShareModule],
|
||||
})
|
||||
export class ShareModule {}
|
||||
@@ -0,0 +1,275 @@
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
import {
|
||||
ProtoServices,
|
||||
ReadService,
|
||||
} from '@dadosfera/protospack-v2/dist/lib/Catalog';
|
||||
import {
|
||||
ForbiddenException,
|
||||
Inject,
|
||||
NotFoundException,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { CatalogClientConfiguration } from '../catalog-client';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { UsersService } from 'src/modules/users/users.service';
|
||||
import { RolesService } from 'src/modules/roles/roles.service';
|
||||
import { RequestUser } from 'src/decorators/user.decorator';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.service';
|
||||
import { isJWT } from 'class-validator';
|
||||
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
|
||||
import { Request } from 'express';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { AuthClientService } from 'src/modules/auth/auth.service';
|
||||
|
||||
|
||||
export class ShareService implements OnModuleInit {
|
||||
catalogReadService: ReadService.CatalogReadServices;
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
@Inject(CatalogClientConfiguration.name)
|
||||
private readonly grpcClient: ClientGrpc,
|
||||
private readonly userService: UsersService,
|
||||
private readonly roleService: RolesService,
|
||||
private readonly shareMetadataService: ShareMetadataService,
|
||||
private readonly mixpanelService: MixpanelService,
|
||||
private authClient: AuthClientService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
this.catalogReadService =
|
||||
this.grpcClient.getService<ReadService.CatalogReadServices>(
|
||||
ProtoServices.CatalogReadServices,
|
||||
);
|
||||
}
|
||||
|
||||
async getDatasetColumnsMetadata(id: string, request: Request) {
|
||||
const shareMetadata = await this.getShareMetadata(id, request);
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id: shareMetadata.customerId,
|
||||
customer_name: shareMetadata.customerName,
|
||||
});
|
||||
const { columns_metadata } = await lastValueFrom(
|
||||
this.catalogReadService.GetDatasetColumnsMetadata(
|
||||
{ id: shareMetadata.assetId, type: undefined },
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
const result = JSON.parse(columns_metadata);
|
||||
return result;
|
||||
}
|
||||
|
||||
async getDatasetPreview(id: string, request: Request) {
|
||||
const shareMetadata = await this.getShareMetadata(id, request);
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id: shareMetadata.customerId,
|
||||
customer_name: shareMetadata.customerName,
|
||||
});
|
||||
const { preview } = await lastValueFrom(
|
||||
this.catalogReadService.GetDatasetPreview(
|
||||
{ id: shareMetadata.assetId, type: undefined },
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
const result = JSON.parse(preview);
|
||||
return result;
|
||||
}
|
||||
|
||||
async getOneDataAssetPublic(id: string, request: Request) {
|
||||
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
|
||||
id
|
||||
}))
|
||||
try {
|
||||
const user = await this.getUserFromRequest(request);
|
||||
|
||||
const shareMetadata = await this.getShareMetadata(id, request);
|
||||
|
||||
const mixpanelTracker = {
|
||||
asset: shareMetadata.assetId,
|
||||
type: isJWT(id) ? 'assigned' : shareMetadata.type,
|
||||
customer: shareMetadata.customerName
|
||||
}
|
||||
|
||||
if (user) {
|
||||
await this.mixpanelService.track("share_page", user, request, mixpanelTracker);
|
||||
} else {
|
||||
await this.mixpanelService.trackShare(request, mixpanelTracker);
|
||||
}
|
||||
|
||||
this.logger.info("shareMetadata: " + JSON.stringify(shareMetadata))
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id: shareMetadata.customerId,
|
||||
customer_name: shareMetadata.customerName,
|
||||
});
|
||||
|
||||
const { data_asset } = await this.getOneDataAsset({
|
||||
customer_id: shareMetadata.customerId,
|
||||
id: shareMetadata.assetId,
|
||||
metadata,
|
||||
});
|
||||
this.logger.info('found asset: ' + JSON.stringify(data_asset));
|
||||
delete data_asset.p_roles;
|
||||
delete data_asset.p_users;
|
||||
data_asset.share_type = 'public';
|
||||
if (data_asset.share_type !== 'public') throw new NotFoundException();
|
||||
|
||||
return { data_asset };
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private async getOneDataAsset(data: {
|
||||
id: string;
|
||||
customer_id: string;
|
||||
metadata: Metadata;
|
||||
}) {
|
||||
const { customer_id, id, metadata } = data;
|
||||
const { data_asset } = await lastValueFrom(
|
||||
this.catalogReadService.GetOneDataAsset(
|
||||
{ id, type: undefined },
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
let asset = JSON.parse(data_asset);
|
||||
asset = {
|
||||
...asset,
|
||||
p_roles: asset.roles,
|
||||
p_users: asset.users,
|
||||
};
|
||||
asset = await this.getAssetsUsersAndRoles([asset], customer_id);
|
||||
|
||||
return { data_asset: asset[0] };
|
||||
}
|
||||
|
||||
async getDataDocs(id: string, request: Request) {
|
||||
const shareMetadata = await this.getShareMetadata(id, request);
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id: shareMetadata.customerId,
|
||||
customer_name: shareMetadata.customerName,
|
||||
});
|
||||
|
||||
const { documentation } = await lastValueFrom(
|
||||
this.catalogReadService.GetDatasetDoc({ id }, metadata),
|
||||
);
|
||||
console.log(documentation);
|
||||
const docs = JSON.parse(documentation);
|
||||
return docs;
|
||||
}
|
||||
|
||||
private async getAssetsUsersAndRoles(
|
||||
data_assets: Array<any>,
|
||||
customer_id: string,
|
||||
) {
|
||||
const { users: customer_users } =
|
||||
await this.userService.findAllUsersByCustomerId(customer_id);
|
||||
const { roles: customer_roles } = await this.roleService.roleSearch(
|
||||
{},
|
||||
{ customer_id },
|
||||
);
|
||||
return data_assets.map((data_asset) => {
|
||||
const owner = customer_users.find(
|
||||
(u) => u.id === data_asset.owner,
|
||||
)?.email;
|
||||
|
||||
const roles = [];
|
||||
const users = [];
|
||||
for (const role_id of data_asset.roles) {
|
||||
const role = customer_roles.find((r) => r.id === role_id);
|
||||
if (role) roles.push({ id: role.id, name: role.name });
|
||||
}
|
||||
for (const user_id of data_asset.users) {
|
||||
const user = customer_users.find((r) => r.id === user_id);
|
||||
if (user) users.push({ id: user.id, email: user.email });
|
||||
}
|
||||
return {
|
||||
...data_asset,
|
||||
roles,
|
||||
users,
|
||||
owner,
|
||||
} as typeof data_asset;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
private async getShareMetadata(id: string, request: Request) {
|
||||
const metadata = PackTheMetadata({});
|
||||
this.logger.info('GET share metadata')
|
||||
const info = await this.shareMetadataService.get(id, metadata);
|
||||
if (isJWT(id) && info ){
|
||||
return info;
|
||||
}
|
||||
|
||||
const user = await this.getUserFromRequest(request);
|
||||
|
||||
if (info.type === 'private') {
|
||||
if (!user) {
|
||||
throw new ForbiddenException(
|
||||
'You do not have permission to access this data asset.',
|
||||
);
|
||||
}
|
||||
|
||||
const is_data_manager = user.permissions.includes(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER.seqid,
|
||||
);
|
||||
|
||||
const is_get = user.permissions.includes(
|
||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET.seqid,
|
||||
);
|
||||
|
||||
if (is_data_manager || is_get) {
|
||||
return info;
|
||||
}
|
||||
|
||||
throw new ForbiddenException(
|
||||
'You do not have permission to access this data asset.',
|
||||
);
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
|
||||
const accessToken = request.get('Authorization');
|
||||
if (accessToken) {
|
||||
const accessTokenDecoded: any = jwt.decode(accessToken, {
|
||||
complete: true,
|
||||
});
|
||||
|
||||
const { kid } = accessTokenDecoded.header;
|
||||
|
||||
const { keys } = await this.authClient.getPublicKeys();
|
||||
|
||||
const pemValue = keys.find((key) => key.kid === kid);
|
||||
|
||||
if (!pemValue) {
|
||||
return null;
|
||||
}
|
||||
|
||||
jwt.verify(accessToken, pemValue.pem);
|
||||
const accessTokenPayload = accessTokenDecoded.payload;
|
||||
|
||||
return {
|
||||
user_id: accessTokenPayload.user_id,
|
||||
username: accessTokenPayload.username,
|
||||
permissions: accessTokenPayload.permissions,
|
||||
customer_id: accessTokenPayload.customer_id,
|
||||
customer_name: accessTokenPayload.customer_name,
|
||||
customer_tier: accessTokenPayload.customer_tier,
|
||||
customer_modules: accessTokenPayload.customer_modules,
|
||||
access_token: accessToken,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
DatabaseConnectionPropertiesDto,
|
||||
} from '../connection/dtos/connection';
|
||||
import { RequestUser } from 'src/decorators/user.decorator';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
|
||||
@Injectable()
|
||||
export class ConnectionTestService {
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
UpdateConnectionDto,
|
||||
} from './dtos/connection';
|
||||
import { CreateConnectionDto } from './dtos/connection';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
|
||||
@@ -25,7 +25,7 @@ import { CustomersService } from './customers.service';
|
||||
import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import type { StringValue } from 'ms';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { EnforceMfa } from './dtos/enforce-mfa';
|
||||
|
||||
@ApiTags('Customers')
|
||||
@@ -136,4 +136,32 @@ export class CustomersController {
|
||||
const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Get(':id/organization-info')
|
||||
@Authenticated()
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
@ApiOkResponse({ description: 'Organization information' })
|
||||
async getOrganizationInfo(@Param('id') id: string) {
|
||||
this.logger.info('getOrganizationInfo', { id });
|
||||
return this.customersService.getOrganizationInfo(id);
|
||||
}
|
||||
|
||||
@Put(':id/organization-info')
|
||||
@Authenticated()
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiOkResponse({ description: 'Organization information updated' })
|
||||
async updateOrganizationInfo(
|
||||
@Param('id') id: string,
|
||||
@Body() body: {
|
||||
companyName: string;
|
||||
companySite: string;
|
||||
domain: string;
|
||||
cnpj: string;
|
||||
description: string;
|
||||
},
|
||||
) {
|
||||
return this.customersService.updateOrganizationInfo(id, body);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -59,6 +59,14 @@ export class CustomersService implements OnModuleInit {
|
||||
);
|
||||
}
|
||||
|
||||
async getCustomer(customerId: string) {
|
||||
return await lastValueFrom(
|
||||
this.customerService.CustomerFindOneById({
|
||||
id: customerId
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
async getLinks(customerId: string) {
|
||||
try {
|
||||
const result = await lastValueFrom(
|
||||
@@ -141,6 +149,7 @@ export class CustomersService implements OnModuleInit {
|
||||
// const decoded = jwt.decode(jwt_token, { complete: true });
|
||||
return jwt_token;
|
||||
}
|
||||
|
||||
async getMonitoringDashboardUrl(metadata: Metadata) {
|
||||
logger.info('CustomersService - getMonitoringDashboardUrl');
|
||||
|
||||
@@ -158,6 +167,7 @@ export class CustomersService implements OnModuleInit {
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async getLogsDashboardUrl(metadata: Metadata) {
|
||||
logger.info('CustomersService - getMixPanelLogsDashboardUrl');
|
||||
|
||||
@@ -213,4 +223,57 @@ export class CustomersService implements OnModuleInit {
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async updateOrganizationInfo(
|
||||
customerId: string,
|
||||
data: {
|
||||
companyName: string;
|
||||
companySite: string;
|
||||
domain: string;
|
||||
cnpj: string;
|
||||
description: string;
|
||||
},
|
||||
) {
|
||||
try {
|
||||
const result = await lastValueFrom(
|
||||
this.customerService.OrganizationUpdate({
|
||||
customerId,
|
||||
companyName: data.companyName || '',
|
||||
companySite: data.companySite || '',
|
||||
domain: data.domain || '',
|
||||
cnpj: data.cnpj || '',
|
||||
description: data.description || '',
|
||||
}),
|
||||
);
|
||||
|
||||
return result;
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async getOrganizationInfo(customerId: string) {
|
||||
try {
|
||||
const customerResponse = await lastValueFrom(
|
||||
this.customerService.CustomerFindOneById({ id: customerId })
|
||||
);
|
||||
|
||||
const customer = customerResponse.customer;
|
||||
|
||||
return {
|
||||
companyName: customer.companyName || '',
|
||||
companySite: customer.companySite || '',
|
||||
domain: customer.domain || '',
|
||||
cnpj: customer.cnpj || '',
|
||||
description: customer.description || ''
|
||||
};
|
||||
} catch (err) {
|
||||
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
|
||||
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
|
||||
else throw err;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class OrganizationUpdateRequest {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@ApiPropertyOptional()
|
||||
companySite: string;
|
||||
@ApiProperty()
|
||||
domain: string;
|
||||
@ApiPropertyOptional()
|
||||
info: string;
|
||||
@ApiPropertyOptional()
|
||||
cnpj: string;
|
||||
}
|
||||
|
||||
export class OrganizationResponse {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@ApiPropertyOptional()
|
||||
companySite: string;
|
||||
@ApiProperty()
|
||||
domain: string;
|
||||
@ApiPropertyOptional()
|
||||
info: string;
|
||||
@ApiPropertyOptional()
|
||||
cnpj: string;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { ApiProperty } from "@nestjs/swagger";
|
||||
|
||||
export class CreateIdentityProvider {
|
||||
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
|
||||
@ApiProperty()
|
||||
clientId: string;
|
||||
|
||||
@ApiProperty()
|
||||
clientSecret: string;
|
||||
|
||||
@ApiProperty()
|
||||
issuerUrl: string;
|
||||
|
||||
@ApiProperty()
|
||||
permissions: number[];
|
||||
}
|
||||
|
||||
|
||||
export class IdentityProviderResponse {
|
||||
|
||||
@ApiProperty()
|
||||
id: string;
|
||||
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
|
||||
@ApiProperty()
|
||||
clientId: string;
|
||||
|
||||
@ApiProperty()
|
||||
issueUrl: string;
|
||||
|
||||
@ApiProperty()
|
||||
permissions: {
|
||||
id: number;
|
||||
name: string;
|
||||
}[];
|
||||
}
|
||||
|
||||
export class IdentityProviderListResponse {
|
||||
|
||||
@ApiProperty()
|
||||
providers: IdentityProviderResponse[]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
export class SsoSignInDto {
|
||||
readonly nonce: string;
|
||||
readonly codeVerifier: string;
|
||||
readonly state: string;
|
||||
readonly id: string;
|
||||
readonly clientId: string;
|
||||
readonly clientSecret: string;
|
||||
readonly issuerUrl: string;
|
||||
readonly redirectUrls: string[];
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
Delete,
|
||||
Get,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Inject,
|
||||
Param,
|
||||
Post,
|
||||
Put,
|
||||
Redirect,
|
||||
Req,
|
||||
} from '@nestjs/common';
|
||||
import { IdentityProviderService } from './identity-provider.service';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { ApiOkResponse } from '@nestjs/swagger';
|
||||
import {
|
||||
CreateIdentityProvider,
|
||||
IdentityProviderListResponse,
|
||||
IdentityProviderResponse,
|
||||
} from './dto/identity-provider.dto';
|
||||
import { Request } from 'express';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import {
|
||||
Authenticated,
|
||||
RequireModule,
|
||||
RequireSomePermission,
|
||||
} from 'src/decorators/authentication.decorator';
|
||||
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||
|
||||
@Controller('identity-providers')
|
||||
export class IdentityProviderController {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private identityProviderService: IdentityProviderService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Post()
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiOkResponse({ type: IdentityProviderResponse })
|
||||
@Authenticated()
|
||||
@RequireModule('sso')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
async addIdentityProvider(
|
||||
@User() user: RequestUser,
|
||||
@Body() body: CreateIdentityProvider,
|
||||
@Language() language: LanguageEnum,
|
||||
) {
|
||||
this.logger.info('POST /identity-providers');
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
...user,
|
||||
language,
|
||||
});
|
||||
|
||||
return await this.identityProviderService.create(body, metadata);
|
||||
}
|
||||
|
||||
@Get()
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ApiOkResponse({ type: IdentityProviderListResponse })
|
||||
@Authenticated()
|
||||
@RequireModule('sso')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
async getProviders(
|
||||
@User() user: RequestUser,
|
||||
@Language() language: LanguageEnum,
|
||||
) {
|
||||
this.logger.info('GET identity-providers');
|
||||
const metadata = PackTheMetadata({
|
||||
...user,
|
||||
language,
|
||||
});
|
||||
|
||||
const result = await this.identityProviderService.getList(metadata);
|
||||
return result;
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@RequireModule('sso')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
async deleteIdentityProvider(
|
||||
@Param('id') id: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
this.logger.info('DELETE /identity-providers');
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
...user,
|
||||
});
|
||||
|
||||
return await this.identityProviderService.deleteIdentityProvider(
|
||||
id,
|
||||
metadata,
|
||||
);
|
||||
}
|
||||
|
||||
@Put(':id')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Authenticated()
|
||||
@RequireModule('sso')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
|
||||
async updateIdentityProviders(
|
||||
@Param('id') id: string,
|
||||
@Body() body: CreateIdentityProvider,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
this.logger.info('PUT /identity-providers');
|
||||
|
||||
const metadata = PackTheMetadata({
|
||||
...user,
|
||||
});
|
||||
|
||||
return await this.identityProviderService.updateIdentityProviders(
|
||||
id,
|
||||
body,
|
||||
metadata,
|
||||
);
|
||||
}
|
||||
|
||||
@Post('/callback')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async callbackIdp(
|
||||
@Req() req: Request,
|
||||
@Language() language: LanguageEnum,
|
||||
@Body()
|
||||
body: {
|
||||
state: string;
|
||||
code: string;
|
||||
},
|
||||
) {
|
||||
this.logger.info('GET /identity-providers/callback');
|
||||
const { code, state } = body;
|
||||
|
||||
if (!code) {
|
||||
this.logger.error('No code received from IDP');
|
||||
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
|
||||
}
|
||||
|
||||
if (!state) {
|
||||
this.logger.error('No state received from IDP');
|
||||
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_RESPONSE);
|
||||
}
|
||||
|
||||
try {
|
||||
const origin = req.headers['origin'] as string;
|
||||
this.logger.info('Header Origin: ' + origin);
|
||||
|
||||
const lang =
|
||||
language.substring(0, 2) + language.substring(2).toUpperCase();
|
||||
const callbackUrl =
|
||||
process.env.ENV !== 'prd'
|
||||
? `${origin}/auth/callback`
|
||||
: `${origin}/${lang}/auth/callback`;
|
||||
|
||||
this.logger.info('Callback URL: ' + callbackUrl);
|
||||
return await this.identityProviderService.getTokenByIdp(
|
||||
code,
|
||||
state,
|
||||
callbackUrl,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@Get('/links')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
async providerLinks(@Req() req: Request) {
|
||||
this.logger.info('GET /identity-providers/links');
|
||||
|
||||
try {
|
||||
const frontDomain = req.headers['origin'] as string;
|
||||
this.logger.info('Header Origin: ' + frontDomain);
|
||||
|
||||
if (!frontDomain) {
|
||||
this.logger.info('Not found front domain');
|
||||
throw new Error(ErrorCodes.IDENTITY_PROVIDER.INVALID_HEADER);
|
||||
}
|
||||
|
||||
const result =
|
||||
await this.identityProviderService.identityProvidersLinksPerDomain(
|
||||
frontDomain,
|
||||
);
|
||||
return result;
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Redirect()
|
||||
async loginIdp(
|
||||
@Param('id') id: string,
|
||||
@Req() req: Request,
|
||||
@Language() language: LanguageEnum,
|
||||
) {
|
||||
this.logger.info('GET /identity-providers/:id');
|
||||
|
||||
try {
|
||||
const frontDomain =
|
||||
(req.headers['origin'] as string) || (req.headers['referer'] as string);
|
||||
this.logger.info(`Front domain: ${frontDomain}`);
|
||||
const host =
|
||||
frontDomain.lastIndexOf('/') !== -1
|
||||
? frontDomain.substring(0, frontDomain.lastIndexOf('/'))
|
||||
: frontDomain;
|
||||
|
||||
const lang =
|
||||
language.substring(0, 2) + language.substring(2).toUpperCase();
|
||||
const callbackUrl =
|
||||
process.env.ENV !== 'prd'
|
||||
? `${host}/auth/callback`
|
||||
: `${host}/${lang}/auth/callback`;
|
||||
|
||||
this.logger.info('Callback URL: ' + callbackUrl);
|
||||
const redirectUrl =
|
||||
await this.identityProviderService.loginIdentityProvider(
|
||||
id,
|
||||
callbackUrl,
|
||||
);
|
||||
|
||||
this.logger.info(`Redirecting to: ${redirectUrl}`);
|
||||
return {
|
||||
url: redirectUrl,
|
||||
};
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { IdentityProviderController } from './identity-provider.controller';
|
||||
import { IdentityProviderService } from './identity-provider.service';
|
||||
import { ClientsModule } from '@nestjs/microservices';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
import { ServicesModule } from 'src/services/service.module';
|
||||
|
||||
const client = new DucClient();
|
||||
|
||||
@Module({
|
||||
imports: [ClientsModule.register([client.providerOptions]), ServicesModule],
|
||||
controllers: [IdentityProviderController],
|
||||
providers: [IdentityProviderService, DadosferaLogger],
|
||||
})
|
||||
export class IdentityProviderModule {}
|
||||
@@ -0,0 +1,185 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Inject,
|
||||
Injectable,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { DucClient } from '../duc/client.config';
|
||||
import { ClientGrpc } from '@nestjs/microservices';
|
||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||
import { IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
|
||||
import { lastValueFrom } from 'rxjs';
|
||||
import { IdentityProviderRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { Issuer, generators } from 'openid-client';
|
||||
import { SsoSignInDto } from './dto/sso-signin.dto';
|
||||
import { CacheService } from 'src/services/cache.service';
|
||||
import { CreateIdentityProvider } from './dto/identity-provider.dto';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Injectable()
|
||||
export class IdentityProviderService implements OnModuleInit {
|
||||
private logger: DadosferaLogger;
|
||||
private identityProviderService: IdentityProviderProtoService;
|
||||
constructor(
|
||||
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
|
||||
private readonly cacheService: CacheService<SsoSignInDto>,
|
||||
@Inject(DadosferaLogger)
|
||||
private dadosferaLoggger: DadosferaLogger
|
||||
) {
|
||||
this.logger = dadosferaLoggger.logger;
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
this.identityProviderService =
|
||||
this.grpcClient.getService<IdentityProviderProtoService>(
|
||||
ProtoServices.IdentityProviderProtoService,
|
||||
);
|
||||
}
|
||||
|
||||
async create(body: IdentityProviderRequest, metadata: Metadata) {
|
||||
this.logger.info("Call IdentityProvider GRPC Create")
|
||||
return await lastValueFrom(
|
||||
this.identityProviderService.Create(body, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async getList(metadata: Metadata) {
|
||||
this.logger.info("Call IdentityProvider GRPC GetList")
|
||||
return await lastValueFrom(
|
||||
this.identityProviderService.GetList({}, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async loginIdentityProvider(id: string, callbackUrl: string) {
|
||||
this.logger.info("Call IdentityProvider GRPC FindIdentityProvider with: " + id);
|
||||
const idp = await lastValueFrom(
|
||||
this.identityProviderService.FindIdentityProvider({ id }),
|
||||
);
|
||||
|
||||
this.logger.info("Discovery issueURL: " + idp.issuerUrl)
|
||||
const issuer = await Issuer.discover(idp.issuerUrl);
|
||||
const client = new issuer.Client({
|
||||
client_id: idp.clientId,
|
||||
client_secret: idp.clientSecret,
|
||||
redirect_uris: idp.redirectUrls,
|
||||
response_types: ['code'],
|
||||
});
|
||||
|
||||
this.logger.info("Generate Challenge")
|
||||
const code_verifier: string = generators.codeVerifier();
|
||||
const code_challenge: string = generators.codeChallenge(code_verifier);
|
||||
|
||||
this.logger.info("Generate State")
|
||||
const state = generators.state();
|
||||
|
||||
this.logger.info("Generate Nonce")
|
||||
const nonce = generators.nonce();
|
||||
|
||||
// Using state because it is returned in the callback
|
||||
// and we can use it to retrieve the code_verifier and nonce
|
||||
this.logger.info("Save Login parameters in redis")
|
||||
await this.cacheService.set(state, {
|
||||
codeVerifier: code_verifier,
|
||||
nonce,
|
||||
id: idp.id,
|
||||
state,
|
||||
clientId: idp.clientId,
|
||||
clientSecret: idp.clientSecret,
|
||||
issuerUrl: idp.issuerUrl,
|
||||
redirectUrls: idp.redirectUrls,
|
||||
});
|
||||
|
||||
this.logger.info("Generate Authorization URL")
|
||||
const url = client.authorizationUrl({
|
||||
scope: 'openid email',
|
||||
response_type: 'code',
|
||||
code_challenge,
|
||||
code_challenge_method: 'S256',
|
||||
state,
|
||||
nonce,
|
||||
redirect_uri: callbackUrl,
|
||||
});
|
||||
const idpUrl = url + '&identity_provider=' + idp.name;
|
||||
this.logger.info(idpUrl)
|
||||
return idpUrl;
|
||||
}
|
||||
|
||||
async getTokenByIdp(code: string, state: string, callbackUrl: string) {
|
||||
this.logger.info("Get login parameters in redis")
|
||||
const ssoSign = await this.cacheService.get(state);
|
||||
|
||||
if (!ssoSign) {
|
||||
this.logger.info("Login Parameters Not Found")
|
||||
throw new BadRequestException('SSO sign-in is expired or not found');
|
||||
}
|
||||
|
||||
this.logger.info("Discovery Issue URL: " + ssoSign.issuerUrl)
|
||||
const issuer = await Issuer.discover(ssoSign.issuerUrl);
|
||||
const client = new issuer.Client({
|
||||
client_id: ssoSign.clientId,
|
||||
client_secret: ssoSign.clientSecret,
|
||||
redirect_uris: ssoSign.redirectUrls,
|
||||
});
|
||||
|
||||
const params = client.callbackParams(
|
||||
`${callbackUrl}?code=${code}&state=${state}`,
|
||||
);
|
||||
try {
|
||||
|
||||
this.logger.info("Get Token Set");
|
||||
const tokenSet = await client.callback(callbackUrl, params, {
|
||||
nonce: ssoSign.nonce,
|
||||
code_verifier: ssoSign.codeVerifier,
|
||||
state: ssoSign.state
|
||||
});
|
||||
|
||||
this.logger.info("Delete parameters in redis");
|
||||
await this.cacheService.delete(ssoSign.state);
|
||||
|
||||
this.logger.info("Call IdentityProvider GRPC SignInUser");
|
||||
return await lastValueFrom(
|
||||
this.identityProviderService.SignInUser({
|
||||
accessToken: tokenSet.access_token,
|
||||
idToken: tokenSet.id_token,
|
||||
refreshToken: tokenSet.refresh_token,
|
||||
id: ssoSign.id,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.error(error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async deleteIdentityProvider(id: string, metadata: Metadata) {
|
||||
this.logger.info("Call IdentityProvider GRPC Delete with: " + id)
|
||||
return await lastValueFrom(
|
||||
this.identityProviderService.DeleteIdentityProvider({ id }, metadata),
|
||||
);
|
||||
}
|
||||
|
||||
async updateIdentityProviders(
|
||||
id: string,
|
||||
body: CreateIdentityProvider,
|
||||
metadata: Metadata,
|
||||
) {
|
||||
this.logger.info("Call IdentityProvider GRPC Update with: " + id)
|
||||
return await lastValueFrom(
|
||||
this.identityProviderService.UpdateIdentityProvider(
|
||||
{
|
||||
id,
|
||||
...body,
|
||||
},
|
||||
metadata,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async identityProvidersLinksPerDomain(frontDomain: string) {
|
||||
this.logger.info("Call IdentityProvider GRPC LinksPerDomain with: " + frontDomain)
|
||||
return await lastValueFrom(
|
||||
this.identityProviderService.GetProviderLinksFromDomain({ frontDomain }),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,13 @@ export class TableColumns {
|
||||
@ApiProperty()
|
||||
references: Column[];
|
||||
@ApiProperty()
|
||||
identifier_columns: string[];
|
||||
@ApiProperty()
|
||||
destination: Record<'raw' | 'qualify', {
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
}> | null;
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
}
|
||||
export class AvailableEntity {
|
||||
|
||||
@@ -200,7 +200,7 @@ export class InputsService {
|
||||
}
|
||||
|
||||
async update(id: string, data, info: Info) {
|
||||
this.validateCron({ ...data, info });
|
||||
// this.validateCron({ ...data, info });
|
||||
try {
|
||||
const updateInputResponse: any = await this.OLD_inputClient.update({
|
||||
id,
|
||||
|
||||
@@ -1,101 +1,46 @@
|
||||
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
|
||||
import { init } from 'mixpanel';
|
||||
import { Authenticated } from 'src/decorators/authentication.decorator';
|
||||
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { RequestUser } from 'src/decorators/user.decorator';
|
||||
import { MixpanelService } from './mixpanel.service';
|
||||
import { extractUserFrom } from 'src/authentication/extract-user';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@ApiInternalOnlyController()
|
||||
@Authenticated()
|
||||
@Controller('trackEvent')
|
||||
export class MixpanelController {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject('MIXPANEL_TOKEN')
|
||||
private readonly mixpanelToken: string,
|
||||
) {}
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
private mixpanelService: MixpanelService,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
@Post(':id')
|
||||
async trackEvent(
|
||||
@Param('id') id,
|
||||
@Body() body,
|
||||
@User() user: RequestUser,
|
||||
@Req() request
|
||||
) {
|
||||
this.logger.info(`POST Track Event: ${id}`)
|
||||
delete body.info;
|
||||
const mixpanel = init(this.mixpanelToken);
|
||||
|
||||
const separator = user.username.includes('-') ? '-' : '.';
|
||||
const removeValues = [
|
||||
'.dadosferatech.dadosfera',
|
||||
'.demo.dadosfera',
|
||||
'.dadosferademo',
|
||||
'.dadosferarh.dadosfera',
|
||||
'.dadosferatech.dadosfera2',
|
||||
'.dadosferatech.dadosfera',
|
||||
'.dadosfera.fin',
|
||||
'.dadosferafin.dadosfera',
|
||||
'.praxio.dadosfera',
|
||||
'.dadosfera.tech',
|
||||
'.treinamentos@dadosfera.ai',
|
||||
'.dadosfera2',
|
||||
'.treinamentosfera',
|
||||
'.dadosfera',
|
||||
];
|
||||
const anonymousUser = {
|
||||
username: "anonymous",
|
||||
customer_name: "anonymous"
|
||||
} as RequestUser
|
||||
|
||||
let username = user.username;
|
||||
const hasToken = request.headers['authorization'];
|
||||
|
||||
removeValues.forEach((value) => {
|
||||
username = username.replace(value, '');
|
||||
});
|
||||
const user = hasToken ? extractUserFrom(hasToken) : anonymousUser;
|
||||
|
||||
username = username.split('@')?.[0];
|
||||
username = username.split('+')?.[0];
|
||||
|
||||
let firstName = username
|
||||
.substring(0, username.indexOf(separator))
|
||||
.replace('dadosfera', '');
|
||||
let lastName = username
|
||||
.substring(username.lastIndexOf(separator) + 1)
|
||||
.replace('dadosfera', '');
|
||||
|
||||
if (!firstName) {
|
||||
firstName = lastName;
|
||||
lastName = '';
|
||||
}
|
||||
|
||||
firstName = this.capitalize(firstName);
|
||||
lastName = this.capitalize(lastName);
|
||||
|
||||
await mixpanel.people.set(user.username, {
|
||||
$first_name: firstName,
|
||||
$last_name: lastName,
|
||||
$name: this.getFullName(firstName, lastName),
|
||||
$email: user.username.includes('@')
|
||||
? user.username
|
||||
: user.username + '@dadosfera.ai',
|
||||
customer_name: user.customer_name
|
||||
});
|
||||
|
||||
await mixpanel.track(id, {
|
||||
distinct_id: user.username,
|
||||
customer: user.customer_name,
|
||||
env: process.env.ENV,
|
||||
$ip: request.ip,
|
||||
$os: request.headers['sec-ch-ua-platform'] || '',
|
||||
$browser: request.headers['user-agent'],
|
||||
...body,
|
||||
});
|
||||
this.logger.info(`Has user: ${typeof hasToken == "string"}`)
|
||||
|
||||
await this.mixpanelService.track(id, user, request, body)
|
||||
|
||||
this.logger.info(`Event successful`)
|
||||
return { id, body, user: user.username };
|
||||
}
|
||||
|
||||
capitalize(sentence: string): string {
|
||||
if (!sentence) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return sentence[0].toUpperCase() + sentence.substring(1);
|
||||
}
|
||||
|
||||
getFullName(firstName: string, lastName: string) {
|
||||
return `${firstName}${lastName ? ' ' + lastName : ''}`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
|
||||
import { MixpanelController } from './mixpanel.controller';
|
||||
import { MixpanelService } from './mixpanel.service';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
|
||||
@Module({
|
||||
controllers: [MixpanelController],
|
||||
@@ -8,9 +10,12 @@ import { MixpanelController } from './mixpanel.controller';
|
||||
{
|
||||
provide: 'MIXPANEL_TOKEN',
|
||||
useValue: getSecretFromSecretsManager(
|
||||
`${process.env.ENV}/root/mixpanel_token`,
|
||||
`prd/root/mixpanel_token`,
|
||||
),
|
||||
},
|
||||
MixpanelService,
|
||||
DadosferaLogger
|
||||
],
|
||||
exports: [MixpanelService]
|
||||
})
|
||||
export class MixpanelModule {}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||
import { Inject } from '@nestjs/common';
|
||||
import { Request } from 'express';
|
||||
import mixpanel, { init } from 'mixpanel';
|
||||
import { RequestUser } from 'src/decorators/user.decorator';
|
||||
|
||||
export class MixpanelService {
|
||||
logger: DadosferaLogger;
|
||||
|
||||
constructor(
|
||||
@Inject('MIXPANEL_TOKEN')
|
||||
private readonly mixpanelToken: string,
|
||||
@Inject(DadosferaLogger)
|
||||
dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async track(eventName: string, user: RequestUser, request: Request, body: any) {
|
||||
this.logger.info("track: " + JSON.stringify({
|
||||
eventName,
|
||||
...body
|
||||
}))
|
||||
const mixpanel = init(this.mixpanelToken);
|
||||
await this.setPeople(user, mixpanel);
|
||||
|
||||
await mixpanel.track(eventName, {
|
||||
distinct_id: user.username,
|
||||
customer: user.customer_name,
|
||||
env: process.env.ENV,
|
||||
$ip: request.ip,
|
||||
$os: request.headers['sec-ch-ua-platform'] || '',
|
||||
$browser: request.headers['user-agent'],
|
||||
...body,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
async trackShare(request: Request, body: any) {
|
||||
this.logger.info("trackShare: " + JSON.stringify(body))
|
||||
const mixpanel = init(this.mixpanelToken);
|
||||
|
||||
await mixpanel.track("share_page", {
|
||||
env: process.env.ENV,
|
||||
$ip: request.ip,
|
||||
$os: request.headers['sec-ch-ua-platform'] || '',
|
||||
$browser: request.headers['user-agent'],
|
||||
...body,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
private async setPeople(user: RequestUser, mixpanel: mixpanel.Mixpanel) {
|
||||
const separator = user.username.includes('-') ? '-' : '.';
|
||||
const removeValues = [
|
||||
'.dadosferatech.dadosfera',
|
||||
'.demo.dadosfera',
|
||||
'.dadosferademo',
|
||||
'.dadosferarh.dadosfera',
|
||||
'.dadosferatech.dadosfera2',
|
||||
'.dadosferatech.dadosfera',
|
||||
'.dadosfera.fin',
|
||||
'.dadosferafin.dadosfera',
|
||||
'.praxio.dadosfera',
|
||||
'.dadosfera.tech',
|
||||
'.treinamentos@dadosfera.ai',
|
||||
'.dadosfera2',
|
||||
'.treinamentosfera',
|
||||
'.dadosfera',
|
||||
];
|
||||
|
||||
let username = user.username;
|
||||
|
||||
removeValues.forEach((value) => {
|
||||
username = username.replace(value, '');
|
||||
});
|
||||
|
||||
username = username.split('@')?.[0];
|
||||
username = username.split('+')?.[0];
|
||||
|
||||
let firstName = username
|
||||
.substring(0, username.indexOf(separator))
|
||||
.replace('dadosfera', '');
|
||||
let lastName = username
|
||||
.substring(username.lastIndexOf(separator) + 1)
|
||||
.replace('dadosfera', '');
|
||||
|
||||
if (!firstName) {
|
||||
firstName = lastName;
|
||||
lastName = '';
|
||||
}
|
||||
|
||||
firstName = this.capitalize(firstName);
|
||||
lastName = this.capitalize(lastName);
|
||||
|
||||
await mixpanel.people.set(user.username, {
|
||||
$first_name: firstName,
|
||||
$last_name: lastName,
|
||||
$name: this.getFullName(firstName, lastName),
|
||||
$email: user.username.includes('@')
|
||||
? user.username
|
||||
: user.username + '@dadosfera.ai',
|
||||
customer_name: user.customer_name,
|
||||
});
|
||||
}
|
||||
|
||||
private capitalize(sentence: string): string {
|
||||
if (!sentence) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return sentence[0].toUpperCase() + sentence.substring(1);
|
||||
}
|
||||
|
||||
private getFullName(firstName: string, lastName: string) {
|
||||
return `${firstName}${lastName ? ' ' + lastName : ''}`;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
import { NetworkPoliciesDTO } from './dto/network-policy.dto';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { NetworkPolicyService } from './network-policy.service';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
|
||||
@Controller('network-policy')
|
||||
export class NetworkPolicyController {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport';
|
||||
import { ConnectionClientService } from '../connection/client.service';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
|
||||
@ApiTags('oauth')
|
||||
@Controller('oauth')
|
||||
|
||||
@@ -9,7 +9,7 @@ import { Language } from 'src/decorators/language.decorator';
|
||||
import { OpenDataService } from './open-data.service';
|
||||
import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
import { request } from 'http';
|
||||
import { Request } from 'express';
|
||||
|
||||
@@ -45,11 +45,11 @@ export class OpenDataController {
|
||||
) {
|
||||
this.logger.info('createUser for open data' + JSON.stringify(request.headers));
|
||||
|
||||
const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
|
||||
if (!corslist.includes(origin)) {
|
||||
this.logger.info('block request by cors list: '+ origin);
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
// const corslist = ["https://devsbm.dadosfera.io", "https://sharingoceandata.com"];
|
||||
// if (!corslist.includes(origin)) {
|
||||
// this.logger.info('block request by cors list: '+ origin);
|
||||
// throw new ForbiddenException();
|
||||
// }
|
||||
|
||||
const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID;
|
||||
const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID;
|
||||
|
||||
@@ -3,6 +3,7 @@ import { ApiOperation, ApiTags } from '@nestjs/swagger';
|
||||
import {
|
||||
AuthenticateCondition,
|
||||
Authenticated,
|
||||
RequireSomePermission,
|
||||
} from 'src/decorators/authentication.decorator';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { PipelinesService } from './pipelines.service';
|
||||
@@ -13,26 +14,6 @@ import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
|
||||
@ApiTags('Pipelines')
|
||||
@Controller('pipelines')
|
||||
@Authenticated()
|
||||
@AuthenticateCondition((req, user) => {
|
||||
let action;
|
||||
|
||||
switch (req.method) {
|
||||
case 'POST':
|
||||
action = 'CREATE';
|
||||
break;
|
||||
|
||||
case 'PUT':
|
||||
action = 'UPDATE';
|
||||
break;
|
||||
|
||||
default:
|
||||
action = req.method;
|
||||
}
|
||||
|
||||
return user.permissions.includes(
|
||||
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
|
||||
);
|
||||
})
|
||||
export class PipelinesController {
|
||||
logger: DadosferaLogger;
|
||||
constructor(
|
||||
@@ -44,6 +25,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Post('start/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
@ApiOperation({
|
||||
deprecated: true,
|
||||
description:
|
||||
@@ -71,6 +53,7 @@ export class PipelinesController {
|
||||
description:
|
||||
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
|
||||
})
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||
body.id = id;
|
||||
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
|
||||
import { Info } from '@dadosfera/protospack/dist/lib/interfaces';
|
||||
|
||||
export class PipelineInputsDTO {
|
||||
@ApiProperty()
|
||||
tables: Array<{
|
||||
name: string,
|
||||
type: string,
|
||||
|
||||
}>
|
||||
}
|
||||
|
||||
export class IPipelineV2 {
|
||||
@ApiProperty()
|
||||
id: string;
|
||||
@@ -123,3 +132,30 @@ export class PipelineFindAllReq {
|
||||
@ApiPropertyOptional()
|
||||
type?: string | undefined;
|
||||
}
|
||||
|
||||
export interface UpdateTableDTO {
|
||||
name: string;
|
||||
type: string;
|
||||
columns: string[];
|
||||
destinations: {
|
||||
raw: {
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
};
|
||||
qualify: {
|
||||
table_schema: string;
|
||||
table_name: string;
|
||||
};
|
||||
};
|
||||
identifier_columns: string[];
|
||||
reference_column: {
|
||||
name: string;
|
||||
type: string;
|
||||
};
|
||||
memory: number;
|
||||
}
|
||||
|
||||
export interface UpdatePlatformInputRequest {
|
||||
cron: string;
|
||||
tables: Array<UpdateTableDTO>;
|
||||
}
|
||||
|
||||
@@ -26,13 +26,14 @@ import {
|
||||
import {
|
||||
AuthenticateCondition,
|
||||
RequireAllPermissions,
|
||||
RequireSomePermission,
|
||||
} from 'src/decorators/authentication.decorator';
|
||||
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
||||
import { PipelinesService } from './pipelines.service';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
|
||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
|
||||
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
|
||||
|
||||
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
|
||||
import {
|
||||
@@ -42,40 +43,20 @@ import {
|
||||
IPipelineV2,
|
||||
IInitUploadCSVFile,
|
||||
PipelineFindAllReq,
|
||||
UpdatePlatformInputRequest,
|
||||
} from './interfaces';
|
||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||
import { LanguageEnum } from 'src/utils/languages.enum';
|
||||
import { Language } from 'src/decorators/language.decorator';
|
||||
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
|
||||
import { TableColumns } from '../inputs/dtos/input.model';
|
||||
import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
|
||||
@ApiTags('PipelinesV2')
|
||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||
@Controller('pipelinesV2')
|
||||
@AuthenticateCondition((req, user) => {
|
||||
let action;
|
||||
|
||||
switch (req.method) {
|
||||
case 'POST':
|
||||
action = 'CREATE';
|
||||
break;
|
||||
|
||||
case 'PUT':
|
||||
action = 'UPDATE';
|
||||
break;
|
||||
|
||||
case 'PATCH':
|
||||
action = 'UPDATE';
|
||||
break;
|
||||
|
||||
default:
|
||||
action = req.method;
|
||||
}
|
||||
|
||||
return user.permissions.includes(
|
||||
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
|
||||
);
|
||||
})
|
||||
export class PipelinesController {
|
||||
logger: DadosferaLogger;
|
||||
constructor(
|
||||
@@ -88,6 +69,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Get('monitoring-dashboard')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getMonitoringDashboard(@User() user: RequestUser) {
|
||||
this.logger.info('PipelinesController - getMonitoringDashboard', { user });
|
||||
|
||||
@@ -100,6 +82,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Post()
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
@ApiCreatedResponse({ type: IPipelineV2 })
|
||||
async create(
|
||||
@Language() language: LanguageEnum,
|
||||
@@ -126,6 +109,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Get()
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async findAll(
|
||||
@User() user: RequestUser,
|
||||
@Language() language: LanguageEnum,
|
||||
@@ -150,6 +134,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Get('/download-logs')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async downloadLogs(
|
||||
@User() user: RequestUser,
|
||||
@Language() language: LanguageEnum,
|
||||
@@ -180,6 +165,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Get(':id/config')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW,PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineproperties(
|
||||
@Language() language: LanguageEnum,
|
||||
@User() user: RequestUser,
|
||||
@@ -191,6 +177,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Get(':id/objects')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineObjects(
|
||||
@Language() language: LanguageEnum,
|
||||
@User() user: RequestUser,
|
||||
@@ -202,16 +189,14 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Get(':id/status')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async getPipelineStatus(@Body() body, @Param('id') id: string) {
|
||||
body.id = id;
|
||||
|
||||
this.logger.info(
|
||||
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
|
||||
{
|
||||
user: body.info.user_id,
|
||||
customer: body.info.customer,
|
||||
},
|
||||
);
|
||||
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
|
||||
user: body.info.user_id,
|
||||
customer: body.info.customer,
|
||||
});
|
||||
|
||||
const response = await this.oldPipelinesService.getPipelineStatus(body);
|
||||
|
||||
@@ -219,6 +204,7 @@ export class PipelinesController {
|
||||
}
|
||||
|
||||
@Get('/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
||||
async findOne(
|
||||
@Language() language: LanguageEnum,
|
||||
@User() user: RequestUser,
|
||||
@@ -241,6 +227,7 @@ export class PipelinesController {
|
||||
.then((res) => {
|
||||
//{pipeline:{tables: {tables: [], input_id: ''}}}
|
||||
let tables = JSON.parse(res.pipeline.config.tables);
|
||||
const input_id = tables?.input_id;
|
||||
if (tables?.tables) tables = tables.tables;
|
||||
Object.assign(res.pipeline, {
|
||||
transformations: res.pipeline.transformations
|
||||
@@ -249,6 +236,7 @@ export class PipelinesController {
|
||||
config: {
|
||||
cron: res.pipeline.config.cron,
|
||||
tables,
|
||||
input_id
|
||||
},
|
||||
properties: res.pipeline.properties
|
||||
? JSON.parse(res.pipeline.properties)
|
||||
@@ -256,10 +244,12 @@ export class PipelinesController {
|
||||
});
|
||||
return res;
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@Patch('/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async update(
|
||||
@Language() language: LanguageEnum,
|
||||
@Body() updatePipelineDto,
|
||||
@@ -293,12 +283,52 @@ export class PipelinesController {
|
||||
return response;
|
||||
}
|
||||
|
||||
@Patch('/:pipelineId/inputs/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updatePipelineInput(
|
||||
@Language() language: LanguageEnum,
|
||||
@Body() pipelineInputDTO: UpdatePlatformInputRequest,
|
||||
@Param('id') inputId: string,
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
this.logger.info('PipelinesController - update', { user });
|
||||
|
||||
|
||||
const { customer_id, customer_name, user_id, username } = user;
|
||||
const info: Info = {
|
||||
user_id: user.user_id,
|
||||
customer: user.customer_name,
|
||||
customer_id: user.customer_id,
|
||||
};
|
||||
const metadata = PackTheMetadata({
|
||||
customer_id,
|
||||
customer_name,
|
||||
user_id,
|
||||
username,
|
||||
language,
|
||||
});
|
||||
|
||||
const response = await this.pipelinesClientService.updatePipelineInput(
|
||||
pipelineId,
|
||||
inputId,
|
||||
pipelineInputDTO,
|
||||
info,
|
||||
user,
|
||||
metadata,
|
||||
);
|
||||
|
||||
this.logger.info('PipelinesController - update: OK', { user });
|
||||
return response;
|
||||
}
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Put('/:id')
|
||||
@ApiOperation({
|
||||
deprecated: true,
|
||||
description: 'This method is deprecated. Please use PATCH instead',
|
||||
})
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||
async updateDeprecated(
|
||||
@Language() language: LanguageEnum,
|
||||
@Body() updatePipelineDto,
|
||||
@@ -314,6 +344,7 @@ export class PipelinesController {
|
||||
@Delete(':id')
|
||||
@ApiNoContentResponse()
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||
async delete(@Param('id') id: string, @User() user: RequestUser) {
|
||||
this.logger.info('PipelinesController - delete', { user });
|
||||
const metadata = PackTheMetadata({
|
||||
@@ -327,7 +358,7 @@ export class PipelinesController {
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Post('/init-upload')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
|
||||
async initUploadFile(
|
||||
@User() user: RequestUser,
|
||||
@Body() body: IInitUploadCSVFile,
|
||||
@@ -359,7 +390,7 @@ export class PipelinesController {
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Post('/complete-upload')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
|
||||
async completeUploadFile(
|
||||
@User() user: RequestUser,
|
||||
@Body() body: ICompleteUploadCSVFile,
|
||||
@@ -377,7 +408,7 @@ export class PipelinesController {
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Post('/file')
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
|
||||
async uploadedFile(
|
||||
@User() user: RequestUser,
|
||||
@Body() body: ICreatePipelineCSVFile,
|
||||
@@ -427,6 +458,7 @@ export class PipelinesController {
|
||||
|
||||
@ApiInternalOnlyEndpoint()
|
||||
@Post('start/:id')
|
||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
|
||||
async activate(@Param('id') id: string, @Body() body) {
|
||||
const { info } = body;
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe
|
||||
import { ConnectorModule } from '../connector/connector.module';
|
||||
import { InputsModule } from '../inputs/inputs.module';
|
||||
import { TransformationsModule } from '../transformations/transformations.module';
|
||||
import { PlatformApiModule } from '../platform-api/platform-api.module';
|
||||
|
||||
const client = new PipelinesClientConfiguration();
|
||||
|
||||
@@ -21,6 +22,7 @@ const client = new PipelinesClientConfiguration();
|
||||
ConnectorModule,
|
||||
InputsModule,
|
||||
TransformationsModule,
|
||||
PlatformApiModule
|
||||
],
|
||||
controllers: [PipelinesController],
|
||||
providers: [PipelinesService, DadosferaLogger],
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
/* eslint-disable no-async-promise-executor */
|
||||
import {
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
@@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs';
|
||||
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||
import { ICreatePipelineV2Req } from './interfaces';
|
||||
import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
||||
import { Metadata } from '@grpc/grpc-js';
|
||||
import { ConnectorClientService } from '../connector/client.service';
|
||||
@@ -26,6 +27,8 @@ import { TransformationsService } from '../transformations/transformations.servi
|
||||
import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
|
||||
import ErrorCodes from 'src/utils/errorCodes';
|
||||
import ErrorBuilder from 'src/utils/ErrorBuilder';
|
||||
import { PlatformApiService } from '../platform-api/platform-api.service';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
|
||||
export class PipelinesService implements OnModuleInit {
|
||||
logger: DadosferaLogger;
|
||||
@@ -39,6 +42,7 @@ export class PipelinesService implements OnModuleInit {
|
||||
private readonly connectorService: ConnectorClientService,
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly transformationsService: TransformationsService,
|
||||
private readonly platformAPI: PlatformApiService
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
@@ -138,6 +142,7 @@ export class PipelinesService implements OnModuleInit {
|
||||
const findOnePipelineResponse = await lastValueFrom(
|
||||
this.pipelineReadService.PipelineV2FindOne(data, metadata),
|
||||
);
|
||||
console.log('pipeline find one response', findOnePipelineResponse);
|
||||
this.logger.info('Done');
|
||||
|
||||
return findOnePipelineResponse;
|
||||
@@ -339,4 +344,137 @@ export class PipelinesService implements OnModuleInit {
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) {
|
||||
this.logger.info('InputClientService - Update');
|
||||
|
||||
this.logger.info('Update Dynamo Reference');
|
||||
const pipelineIdFormat = pipelineId.split('-').join('_');
|
||||
const updateInputResponse = await this.inputsService.update(
|
||||
inputId,
|
||||
updateInputDTO,
|
||||
info
|
||||
)
|
||||
|
||||
const requests = [];
|
||||
|
||||
this.logger.info('Dynamo Response', updateInputResponse);
|
||||
|
||||
for (const [index, table] of updateInputDTO.tables.entries()) {
|
||||
const id = `${pipelineIdFormat}_${index}`;
|
||||
this.logger.info('Updating input reference for table', table.name);
|
||||
const body = {}
|
||||
if (table.columns) {
|
||||
body['column_include_list'] = table.columns;
|
||||
}
|
||||
|
||||
if (table.reference_column) {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
}
|
||||
|
||||
if (table.identifier_columns) {
|
||||
body['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
this.logger.info('Request body', body);
|
||||
const updateCollumns = this.platformAPI.proxy(
|
||||
'PATCH',
|
||||
`/jobs/${id}/input`,
|
||||
user,
|
||||
body
|
||||
)
|
||||
requests.push(updateCollumns);
|
||||
|
||||
if (table.memory) {
|
||||
this.logger.info('Updating memory allocation for table', table.name);
|
||||
const updateMemory = this.platformAPI.proxy(
|
||||
'PUT',
|
||||
`/jobs/${id}/memory`,
|
||||
user,
|
||||
{
|
||||
amount: table.memory
|
||||
}
|
||||
)
|
||||
requests.push(updateMemory);
|
||||
}
|
||||
|
||||
if (table.type) {
|
||||
const updateSyncMode = this.updatePipelineSyncMode(table, id, user);
|
||||
requests.push(updateSyncMode);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.info('Create Platform Request for each JOB');
|
||||
|
||||
if (updateInputDTO.cron) {
|
||||
const crnUpdatedRequest = new Promise(async (resolve, reject) => {
|
||||
const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user);
|
||||
|
||||
if (response.error) {
|
||||
this.logger.error('Error updating pipeline cron', response.error);
|
||||
return reject(new ErrorBuilder(response.error));
|
||||
}
|
||||
this.logger.error('Pipeline cron updated successfully', response);
|
||||
return resolve(response);
|
||||
});
|
||||
requests.push(crnUpdatedRequest);
|
||||
}
|
||||
|
||||
this.logger.info('Executing all request for the platform api');
|
||||
|
||||
const results = await Promise.allSettled(requests);
|
||||
this.logger.info('Platform api response', results);
|
||||
|
||||
return updateInputResponse;
|
||||
|
||||
}
|
||||
|
||||
private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) {
|
||||
const body = {
|
||||
target_load_type: table.type
|
||||
}
|
||||
|
||||
if (table.type === 'incremental_with_qualify') {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
body['primary_keys'] = table.identifier_columns;
|
||||
}
|
||||
|
||||
if (table.type === 'incremental') {
|
||||
body['incremental_column_name'] = table.reference_column.name;
|
||||
body['incremental_column_type'] = table.reference_column.type;
|
||||
}
|
||||
|
||||
this.logger.info('Updating pipeline sync mode', {
|
||||
pipelineId,
|
||||
body
|
||||
});
|
||||
|
||||
return this.platformAPI.proxy(
|
||||
"POST",
|
||||
`/jobs/jdbc/${pipelineId}/sync-mode`,
|
||||
user,
|
||||
body
|
||||
)
|
||||
}
|
||||
|
||||
private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) {
|
||||
try {
|
||||
const response = await this.platformAPI.proxy(
|
||||
'PATCH',
|
||||
`/pipeline/${pipelineId}`,
|
||||
user,
|
||||
{
|
||||
cron
|
||||
}
|
||||
);
|
||||
|
||||
return response
|
||||
} catch (error) {
|
||||
return {
|
||||
error: error.message
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
export const PLATFORM_API_CONFIG = {
|
||||
getUrl: (): string => {
|
||||
const url = process.env.PLATFORM_API_URL;
|
||||
if (!url) {
|
||||
throw new Error('PLATFORM_API_URL environment variable is not set');
|
||||
}
|
||||
return url;
|
||||
},
|
||||
region: process.env.AWS_REGION || 'us-east-1',
|
||||
timeout: parseInt(process.env.PLATFORM_API_TIMEOUT || '30000', 10),
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user