From 7d69220461b8a9ba69e569f90719e1709d136979 Mon Sep 17 00:00:00 2001 From: Rafael Date: Sat, 12 Oct 2024 13:19:57 -0300 Subject: [PATCH 01/45] UPDATE: create helm chart for maestro --- .github/workflows/deploy-k8s.yml | 73 +++++++++++++++++++ .github/workflows/validate-k8s.yml | 79 ++++++++++++++++++++ helmfiles/prd.yaml | 16 +++++ helmfiles/stg.yaml | 16 +++++ maestro/.helmignore | 23 ++++++ maestro/Chart.yaml | 24 +++++++ maestro/templates/NOTES.txt | 0 maestro/templates/deployment.yaml | 112 +++++++++++++++++++++++++++++ maestro/templates/ingress.yaml | 27 +++++++ maestro/templates/secret.yaml | 40 +++++++++++ maestro/templates/service.yaml | 18 +++++ maestro/values.yaml | 49 +++++++++++++ 12 files changed, 477 insertions(+) create mode 100644 .github/workflows/deploy-k8s.yml create mode 100644 .github/workflows/validate-k8s.yml create mode 100644 helmfiles/prd.yaml create mode 100644 helmfiles/stg.yaml create mode 100644 maestro/.helmignore create mode 100644 maestro/Chart.yaml create mode 100644 maestro/templates/NOTES.txt create mode 100644 maestro/templates/deployment.yaml create mode 100644 maestro/templates/ingress.yaml create mode 100644 maestro/templates/secret.yaml create mode 100644 maestro/templates/service.yaml create mode 100644 maestro/values.yaml diff --git a/.github/workflows/deploy-k8s.yml b/.github/workflows/deploy-k8s.yml new file mode 100644 index 0000000..f547ac4 --- /dev/null +++ b/.github/workflows/deploy-k8s.yml @@ -0,0 +1,73 @@ +name: Deploy K8S Modifications + +on: + push: + branches: + - main + - beta + +jobs: + extract_environment: + runs-on: ubuntu-22.04 + outputs: + environment: ${{ steps.extract_environment.outputs.environment }} + steps: + - name: Extract Environment + run: | + if [ ${GITHUB_REF} == "refs/heads/main" ]; then + echo "environment=prd" >> $GITHUB_OUTPUT + elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then + echo "environment=stg" >> $GITHUB_OUTPUT + fi + id: extract_environment + + helmfile-deploy: + needs: [extract_environment] + runs-on: [self-hosted, "prd-azure"] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: 'v3.9.0' + + - name: Install Azure ClI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | bash + + - uses: azure/login@v2 + with: + creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install Helmfile + run: | + wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + tar -xzf helmfile_0.148.0_linux_amd64.tar.gz + mv helmfile /usr/local/bin/ + helmfile --version + + - name: Install Helm Diff Plugin + run: helm plugin install https://github.com/databus23/helm-diff || true + + - name: Setup kubectl + uses: azure/setup-kubectl@v1 + with: + version: 'v1.30.1' + + - name: Authenticate with cluster + env: + CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} + run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing + + - name: Run Helmfile Apply + env: + ENV: ${{ needs.extract_environment.outputs.environment }} + run: helmfile -f helmfiles/${ENV}.yaml sync diff --git a/.github/workflows/validate-k8s.yml b/.github/workflows/validate-k8s.yml new file mode 100644 index 0000000..17ed975 --- /dev/null +++ b/.github/workflows/validate-k8s.yml @@ -0,0 +1,79 @@ +name: Validate K8S Modifications + +on: + pull_request: + branches: + - main + - stg + +jobs: + extract_environment: + runs-on: ubuntu-22.04 + outputs: + environment: ${{ steps.extract_environment.outputs.environment }} + steps: + - name: Extract Environment + run: | + if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then + echo "environment=prd" >> $GITHUB_OUTPUT + elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then + echo "environment=stg" >> $GITHUB_OUTPUT + fi + id: extract_environment + + helmfile-deploy: + needs: [extract_environment] + runs-on: [self-hosted, "prd-azure"] + + steps: + - name: Summary + env: + ENV: ${{ needs.extract_environment.outputs.environment }} + run: | + echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\` para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: 'v3.9.0' + + - name: Install Azure ClI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | bash + + - uses: azure/login@v2 + with: + creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install Helmfile + run: | + wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + tar -xzf helmfile_0.148.0_linux_amd64.tar.gz + mv helmfile /usr/local/bin/ + helmfile --version + + - name: Install Helm Diff Plugin + run: helm plugin install https://github.com/databus23/helm-diff || true + + - name: Setup kubectl + uses: azure/setup-kubectl@v1 + with: + version: 'v1.30.1' + + - name: Authenticate with cluster + env: + CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} + run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing + + - name: Run Helmfile Diff + env: + ENV: ${{ needs.extract_environment.outputs.environment }} + run: helmfile -f helmfiles/${ENV}.yaml diff diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml new file mode 100644 index 0000000..8076b3b --- /dev/null +++ b/helmfiles/prd.yaml @@ -0,0 +1,16 @@ +charts: + - name: maestro + chart: ../maestro + values: + - ../maestro/values.yaml + set: + - name: maestro.duc_url + value: duc.dadosfera.ai + - name: hostname + value: maestro.dadosfera.ai + - name: maestro.pi_factory_url + value: pi-factory.dadosfera.ai + - name: maestro.in_factory_url + value: in-factory.dadosfera.ai + - name: maestro.tr_factory_url + value: in-factory.dadosfera.ai \ No newline at end of file diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml new file mode 100644 index 0000000..eb1fc69 --- /dev/null +++ b/helmfiles/stg.yaml @@ -0,0 +1,16 @@ +charts: + - name: maestro + chart: ../maestro + values: + - ../maestro/values.yaml + set: + - name: maestro.duc_url + value: duc-temp.dadosfera.ai + - name: hostname + value: maestro-temp.dadosfera.ai + - name: maestro.pi_factory_url + value: pi-factory-temp.dadosfera.ai + - name: maestro.in_factory_url + value: in-factory-temp.dadosfera.ai + - name: maestro.tr_factory_url + value: in-factory-temp.dadosfera.ai \ No newline at end of file diff --git a/maestro/.helmignore b/maestro/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/maestro/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/maestro/Chart.yaml b/maestro/Chart.yaml new file mode 100644 index 0000000..c5c75d6 --- /dev/null +++ b/maestro/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: maestro +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/maestro/templates/NOTES.txt b/maestro/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml new file mode 100644 index 0000000..18fad50 --- /dev/null +++ b/maestro/templates/deployment.yaml @@ -0,0 +1,112 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: maestro + namespace: applications + labels: + app: maestro + +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: maestro + + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: maestro + + spec: + imagePullSecrets: + - name: {{ .Values.imagePullSecrets }} + nodeSelector: + "beta.kubernetes.io/os": linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: application + operator: In + values: + - backend + + containers: + - name: maestro + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + ports: + - containerPort: {{ .Values.containerPort }} + resources: + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.memory }} + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }} + env: + - name: AWS_IDENTITY_POOL_ID + value: {{ .Values.maestro.aws_identity_pool_id }} + - name: AWS_REGION + value: "us-east-1" + - name: BASE_HOST + value: "maestro_prd" + - name: BUCKET_CUSTOMER_CSV_ASSETS + value: {{ .Values.maestro.bucket_customer_csv_assets }} + - name: CONNECTORS_INDEX + value: "connectors" + - name: DUC_URL + value: {{ .Values.maestro.duc_url }} + - name: ELASTIC_APM_ENVIRONMENT + value: {{ .Values.maestro.env }} + - name: ELASTIC_APM_SERVER_URL + value: "https://apm-server.dadosfera.ai" + - name: ELASTIC_APM_SERVICE_NAME + value: {{ .Values.maestro.apm_service }} + - name: ENV + value: {{ .Values.maestro.env }} + - name: INFACTORY_URL + value: {{ .Values.maestro.in_factory_url }} + - name: LOGGER_GELF_HOST + value: "logstash-pipelines.dadosfera.ai" + - name: LOGGER_GELF_PORT + value: "{{ .Values.maestro.logger_gelf_port }}" + - name: NIMBUS_BASE_URL + value: "http://nimbus-api" + - name: NPM_TOKEN + value: {{ .Values.maestro.npm_token }} + - name: PB_TOKEN_PATH + value: {{ .Values.maestro.pb_token_path }} + - name: PIFACTORY_URL + value: {{ .Values.maestro.pi_factory_url }} + - name: SM_OAUTH_PATH + value: {{ .Values.maestro.sm_oauth_path }} + - name: TRFACTORY_URL + value: {{ .Values.maestro.tr_factory_url }} + - name: UPLOAD_FILE_AGENT_CONNECTION + value: {{ .Values.maestro.upload_file_agent_connection }} + - name: JWT_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: prd-duc + key: jwt_token + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: prd-maestro + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: prd-maestro + key: AWS_SECRET_ACCESS_KEY + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + name: prd-maestro + key: AWS_DEFAULT_REGION diff --git a/maestro/templates/ingress.yaml b/maestro/templates/ingress.yaml new file mode 100644 index 0000000..faf1305 --- /dev/null +++ b/maestro/templates/ingress.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/server-snippet: | + underscores_in_headers on; + ignore_invalid_headers on; + + generation: 1 + labels: + app: maestro + name: maestro + namespace: applications +spec: + ingressClassName: nginx + rules: + - host: {{ .Values.hostname }} + http: + paths: + - backend: + service: + name: maestro + port: + number: {{ .Values.ingress.port }} + path: / + pathType: Prefix diff --git a/maestro/templates/secret.yaml b/maestro/templates/secret.yaml new file mode 100644 index 0000000..9e2fd37 --- /dev/null +++ b/maestro/templates/secret.yaml @@ -0,0 +1,40 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: prd-maestro + namespace: applications + labels: + app: maestro +spec: + refreshInterval: 1h + secretStoreRef: + name: secretsmanager-prd + kind: SecretStore + target: + name: prd-maestro + creationPolicy: Owner + data: + - secretKey: AWS_ACCESS_KEY_ID + remoteRef: + key: prd/microservices/aws_credentials/maestro + version: "AWSCURRENT" + property: AWS_ACCESS_KEY_ID + + - secretKey: AWS_SECRET_ACCESS_KEY + remoteRef: + key: prd/microservices/aws_credentials/maestro + version: "AWSCURRENT" + property: AWS_SECRET_ACCESS_KEY + + - secretKey: AWS_DEFAULT_REGION + remoteRef: + key: prd/microservices/aws_credentials/maestro + version: "AWSCURRENT" + property: AWS_DEFAULT_REGION + + - secretKey: jwt_token + remoteRef: + key: {{ .Values.maestro.jwt_token_secret_id }} + version: "AWSCURRENT" + property: token + diff --git a/maestro/templates/service.yaml b/maestro/templates/service.yaml new file mode 100644 index 0000000..75f635a --- /dev/null +++ b/maestro/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: maestro + namespace: applications + labels: + app: maestro + +spec: + type: ClusterIP + ports: + - name: maestro + protocol: TCP + port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetPort }} + selector: + app: maestro + diff --git a/maestro/values.yaml b/maestro/values.yaml new file mode 100644 index 0000000..a8d745c --- /dev/null +++ b/maestro/values.yaml @@ -0,0 +1,49 @@ +# Default values for metabase. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 3 +hostname: maestro-temp.dadosfera.ai +image: + repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: 1.56.0 +containerPort: 3333 +imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret" +service: + type: ClusterIP + port: 3333 + targetPort: 3333 +ingress: + enabled: false + port: 3333 +resources: + requests: + cpu: 100m + memory: 1500Mi + limits: + cpu: 2000m + memory: 2Gi +maestro: + aws_identity_pool_id: "us-east-1_Mrezsw9Sn" + duc_url: duc.dadosfera.ai + in_factory_url: in-factory.dadosfera.ai + bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153" + env: prd + apm_service: maestro + jwt_token_secret_id: prd/root/jwt_token + logger_gelf_port: "1026" + npm_token: npm_Xp17h3daMkORcZ55NY95Ez4gRfdzsQ3UvINP + pb_token_path: prd/root/productboard_token + pi_factory_url: pi-factory.dadosfera.ai + sm_oauth_path: prd/root/oauth_applications + tr_factory_url: in-factory.dadosfera.ai + upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: 80 From c10f85950aa8975a66f107c66c735f41b3ef2abd Mon Sep 17 00:00:00 2001 From: Rafael Date: Sat, 12 Oct 2024 13:28:06 -0300 Subject: [PATCH 02/45] UPDATE: Commented out the aws beanstalk deployment --- .github/workflows/deploy-manually.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy-manually.yml b/.github/workflows/deploy-manually.yml index 824a119..b761d43 100644 --- a/.github/workflows/deploy-manually.yml +++ b/.github/workflows/deploy-manually.yml @@ -144,15 +144,15 @@ jobs: sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml zip deploy.zip docker-compose.yml -r .ebextensions - - name: Deploy AWS Beanstalk - env: - ENV: ${{ needs.extract_environment.outputs.environment }} - AWS_REGION: us-east-1 - APP_NAME: ${{ github.event.repository.name }} - run: | - eb use $APP_NAME-$ENV - echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml - eb deploy + # - name: Deploy AWS Beanstalk + # env: + # ENV: ${{ needs.extract_environment.outputs.environment }} + # AWS_REGION: us-east-1 + # APP_NAME: ${{ github.event.repository.name }} + # run: | + # eb use $APP_NAME-$ENV + # echo -e "deploy:\n artifact: deploy.zip" >> .elasticbeanstalk/config.yml + # eb deploy - name: Remove Docker's Trash if: always() From ccdff5805623bd6da765342f100c9ffb357c0e0b Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 4 Dec 2024 15:28:57 -0300 Subject: [PATCH 03/45] FEAT: update protospack lib --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 61c7a9a..329e2bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.34.0", + "@dadosfera/protospack-v2": "3.36.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1403,9 +1403,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.34.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.34.0.tgz", - "integrity": "sha512-VUjpoHg5/uNNkg1cTxWbUv+20t5CTAf1tB4dhBmGSYjgb+Efo5gi3aO/p3ajYuzOOMOxF+EhjQVouU6BD3E+Xg==", + "version": "3.36.0-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.36.0-beta.1.tgz", + "integrity": "sha512-ePznNQnYaxwWGR8hpFDXE35wPql9oPC9vEtwnZw3PzcfKg37vQUKRSzfjP9xyuAeWQND4OhFOz6wrTmS8GFHzQ==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index fbe7eea..7dc5019 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.34.0", + "@dadosfera/protospack-v2": "3.36.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From ae81816aa789d980a84791ee7d0b42f632adabd2 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 4 Dec 2024 15:29:51 -0300 Subject: [PATCH 04/45] FEAT: create get and post route to theme --- docsfera.json | 145 +++++++++++++++++- src/modules/customers/customers.controller.ts | 44 +++++- src/modules/customers/customers.service.ts | 52 ++++++- src/modules/customers/dtos/customers.ts | 29 +++- 4 files changed, 263 insertions(+), 7 deletions(-) diff --git a/docsfera.json b/docsfera.json index 0922da4..381bdfe 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5371,6 +5371,92 @@ ] } }, + "/customers/{id}/theme": { + "post": { + "operationId": "CustomersController_saveCustomertheme", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeRequest" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + }, + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "CustomersController_getCustomerTheme", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + } + }, + "tags": [ + "Customers" + ] + } + }, "/customers/token": { "get": { "operationId": "CustomersController_getCustomerToken", @@ -8221,6 +8307,63 @@ "links" ] }, + "CustomerThemeRequest": { + "type": "object", + "properties": { + "backgroundColor": { + "type": "string" + }, + "textColor": { + "type": "string" + }, + "logoWhite": { + "type": "string" + }, + "logoBlack": { + "type": "string" + } + }, + "required": [ + "backgroundColor", + "textColor", + "logoWhite", + "logoBlack" + ] + }, + "CustomerTheme": { + "type": "object", + "properties": { + "backgroundColor": { + "type": "string" + }, + "textColor": { + "type": "string" + }, + "logoWhite": { + "type": "string" + }, + "logoBlack": { + "type": "string" + } + }, + "required": [ + "backgroundColor", + "textColor", + "logoWhite", + "logoBlack" + ] + }, + "CustomerThemeResponse": { + "type": "object", + "properties": { + "theme": { + "$ref": "#/components/schemas/CustomerTheme" + } + }, + "required": [ + "theme" + ] + }, "CustomerLinkRequest": { "type": "object", "properties": { @@ -8238,4 +8381,4 @@ } } } -} +} \ No newline at end of file diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 04073ec..4268e6a 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -5,9 +5,11 @@ import { Controller, Get, HttpCode, + HttpException, HttpStatus, Inject, Param, + Post, Put, Query, UseFilters, @@ -18,17 +20,16 @@ import { Authenticated, RequireAllPermissions, } from 'src/decorators/authentication.decorator'; -import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { CustomersService } from './customers.service'; -import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers'; +import { CustomerLinkRequest, CustomerLinksResponse, CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; import { RequestUser, User } from 'src/decorators/user.decorator'; import type { StringValue } from 'ms'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import ErrorCodes from 'src/utils/errorCodes'; @ApiTags('Customers') @Controller('customers') -@Authenticated() @UseFilters(GrpcToHttpExceptionFilter) export class CustomersController { logger: DadosferaLogger; @@ -42,6 +43,7 @@ export class CustomersController { } @Get(':id/links') + @Authenticated() @ApiOkResponse({ type: CustomerLinksResponse }) async getCustomerLinks(@Param('id') id: string) { this.logger.info('getCustomerLinks', { id }); @@ -49,7 +51,41 @@ export class CustomersController { return { links }; } + @Post(':id/theme') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse({ type: CustomerThemeResponse }) + async saveCustomertheme(@Param('id') id: string, @Body() data: CustomerThemeRequest) { + this.logger.info('saveCustomertheme', JSON.stringify({ + id, theme: data + })); + const theme = await this.customersService.createThemeByCustomer(id, data); + + return theme; + } + + @Get(':id/theme') + @ApiOkResponse({ type: CustomerThemeResponse }) + async getCustomerTheme(@Param('id') id: string) { + this.logger.info('getCustomerTheme with id' + id); + + try { + const data = await this.customersService.getThemeByCustomer(id); + this.logger.info('Success - getCustomerTheme with id'+ id); + return data; + }catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message); + return { theme: null }; + }; + } + } + @Put(':id/links') + @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse() @HttpCode(HttpStatus.OK) @@ -63,6 +99,7 @@ export class CustomersController { } @Get('token') + @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN) @ApiProduces('text/plain') async getCustomerToken( @@ -79,6 +116,7 @@ export class CustomersController { } @Get('monitoring-dashboard') + @Authenticated() @RequireAllPermissions( PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD, ) diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 5102db5..5dc43f3 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -28,19 +28,25 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { Metadata } from '@grpc/grpc-js'; import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client'; +import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; // This function will accept any string, which may result in a bug. @Injectable() export class CustomersService implements OnModuleInit { private customerService: CustomersProtoService; - + private logger: DadosferaLogger; private pipelineReadService: ReadService.PipelineV2ReadService; constructor( @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, @Inject(PipelinesClientConfiguration.name) private readonly pipelinesGrpcClient: ClientGrpc, - ) {} + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } onModuleInit() { this.customerService = this.grpcClient.getService( @@ -84,6 +90,48 @@ export class CustomersService implements OnModuleInit { } } + async createThemeByCustomer(id: string, theme: CustomerThemeRequest): Promise { + if (!id) { + this.logger.error('Error - saveCustomertheme - not found id:' + id); + throw new HttpException(null, HttpStatus.BAD_REQUEST); + } + + try { + const result = await firstValueFrom( + this.customerService.CustomerCreateTheme({ + customerId: id, + theme + }), + ); + return { theme: result }; + } catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message); + throw err + }; + } + } + + async getThemeByCustomer(id: string): Promise { + if (!id) { + this.logger.error('Error - getCustomerTheme - not found id:' + id); + throw new HttpException(null, HttpStatus.BAD_REQUEST); + } + + const result = await firstValueFrom( + this.customerService.CustomerGetTheme({ + id + }), + ); + + return { + theme: result + } + } + async generateToken( expiresIn = '30m', data: { customerId: string; userId: string; customerName: string }, diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index c3b3070..e26fc94 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -1,4 +1,4 @@ -import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; +import { Link, Theme } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; export class CustomerLink implements Link { @@ -20,3 +20,30 @@ export class CustomerLinksResponse { @ApiProperty({ type: [CustomerLink] }) links: CustomerLink[]; } + +export class CustomerTheme implements Theme { + @ApiProperty() + backgroundColor: string; + @ApiProperty() + textColor: string; + @ApiProperty() + logoWhite: string; + @ApiProperty() + logoBlack: string; +} + +export class CustomerThemeResponse { + @ApiProperty() + theme: CustomerTheme; +} + +export class CustomerThemeRequest { + @ApiProperty() + backgroundColor: string; + @ApiProperty() + textColor: string; + @ApiProperty() + logoWhite: string; + @ApiProperty() + logoBlack: string; +} From 48d4cbc41a08d125d32d63b1f5c0ffbd0e1671b3 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 5 Dec 2024 10:29:06 -0300 Subject: [PATCH 05/45] CHORE: stable version lib protospack --- package-lock.json | 9 +++++---- package.json | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 329e2bc..ff693d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.36.0-beta.1", + "@dadosfera/protospack-v2": "^3.36.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1403,9 +1403,10 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.36.0-beta.1", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.36.0-beta.1.tgz", - "integrity": "sha512-ePznNQnYaxwWGR8hpFDXE35wPql9oPC9vEtwnZw3PzcfKg37vQUKRSzfjP9xyuAeWQND4OhFOz6wrTmS8GFHzQ==", + "version": "3.36.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.36.1.tgz", + "integrity": "sha512-IbkkjoTjhyhglVt4H78n1atNfC3wl5EDDfwhWA9H6VpHLyj8Fbu4XCnZH839oVC5eDtAaq4f4y8RyDueO/L8zg==", + "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 7dc5019..1daeda0 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.36.0-beta.1", + "@dadosfera/protospack-v2": "^3.36.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From ad03d663bbd8387527556eff74013d55e789b170 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 9 Dec 2024 14:00:56 -0300 Subject: [PATCH 06/45] CI: updating CI to use beta branch --- .github/workflows/deploy-manually.yml | 169 ++++++++++++-------------- build.docker-compose.yml | 2 +- 2 files changed, 82 insertions(+), 89 deletions(-) diff --git a/.github/workflows/deploy-manually.yml b/.github/workflows/deploy-manually.yml index b761d43..87e306e 100644 --- a/.github/workflows/deploy-manually.yml +++ b/.github/workflows/deploy-manually.yml @@ -3,6 +3,7 @@ on: push: branches: - main + - beta workflow_dispatch: inputs: environment: @@ -11,7 +12,6 @@ on: type: choice options: - stg - - stg2 - prd push_to_dockerhub: description: "Push image to Dockerhub?" @@ -34,6 +34,8 @@ jobs: echo "environment=${DEPLOY_ENV}" >> $GITHUB_OUTPUT elif [ ${GITHUB_REF} == "refs/heads/main" ]; then echo "environment=prd" >> $GITHUB_OUTPUT + elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then + echo "environment=stg" >> $GITHUB_OUTPUT fi id: extract_environment @@ -48,38 +50,40 @@ jobs: - if: github.event_name != 'workflow_dispatch' name: Semantic Release - uses: cycjimmy/semantic-release-action@v4 + uses: cycjimmy/semantic-release-action@v3 id: semantic with: extra_plugins: | - conventional-changelog-eslint@4 + conventional-changelog-eslint@4.0.0 branches: | [ - 'main' + 'main', + { + name: 'alpha', + prerelease: true + }, + { + name: 'beta', + prerelease: true + } ] env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - deploy-info: - if: ${{ github.event_name == 'workflow_dispatch'}} + build_ecr_image: + if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }} needs: [extract_environment, semantic_release] - runs-on: ubuntu-latest + runs-on: + [self-hosted, "prd"] steps: - - name: Create summary + - name: Printing stats env: EVENT: ${{ github.event_name }} IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} ENV: ${{ needs.extract_environment.outputs.environment }} - run: echo "### Deploy da branch \`$GITHUB_REF_NAME\` no ambiente **$ENV** :rocket:" >> $GITHUB_STEP_SUMMARY + run: echo ${GITHUB_REF#refs/heads/} - deploy: - if: ${{ github.event_name == 'workflow_dispatch' || needs.semantic_release.outputs.new_release_published == 'true' }} - needs: [extract_environment, semantic_release] - runs-on: - [self-hosted, "${{ needs.extract_environment.outputs.environment }}"] - - steps: - name: Checkout uses: actions/checkout@v4 @@ -131,18 +135,18 @@ jobs: docker compose -f build.docker-compose.dockerhub.yml build docker compose -f build.docker-compose.dockerhub.yml push - - name: Create ZIP file to Deploy AWS Beanstalk - env: - ENV: ${{ needs.extract_environment.outputs.environment }} - IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} - ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} - NODE_EXPORTER_URL: ${{format('{0}.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest', steps.aws.outputs.aws-account-id)}} # ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}} - run: | - sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml - sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml - sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml - sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml - zip deploy.zip docker-compose.yml -r .ebextensions + # - name: Create ZIP file to Deploy AWS Beanstalk + # env: + # ENV: ${{ needs.extract_environment.outputs.environment }} + # IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} + # ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }} + # NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}} + # run: | + # sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml + # sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml + # sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml + # sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml + # zip deploy.zip docker-compose.yml -r .ebextensions # - name: Deploy AWS Beanstalk # env: @@ -159,66 +163,55 @@ jobs: run: | docker system prune --volumes -a -f docker system df - api_docs: - needs: [extract_environment, semantic_release, deploy] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Extract Docs BlockId and PageId + helmfile-deploy: + needs: [extract_environment, semantic_release, build_ecr_image] + runs-on: [self-hosted, "prd-azure"] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: 'v3.9.0' + + - name: Install Azure ClI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | bash + + - uses: azure/login@v2 + with: + creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install Helmfile + run: | + wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + tar -xzf helmfile_0.148.0_linux_amd64.tar.gz + mv helmfile /usr/local/bin/ + helmfile --version + + - name: Install Helm Diff Plugin + run: helm plugin install https://github.com/databus23/helm-diff || true + + - name: Setup kubectl + uses: azure/setup-kubectl@v1 + with: + version: 'v1.30.1' + + - name: Authenticate with cluster + env: + CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} + run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing + + - name: Run Helmfile Apply env: ENV: ${{ needs.extract_environment.outputs.environment }} - STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }} - STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }} - STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }} - STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }} - PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }} - PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }} - shell: bash - run: | - if [ $ENV == "stg2" ]; then - echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT - echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT - elif [ $ENV == "stg" ]; then - echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT - echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT - elif [ $ENV == "prd" ]; then - echo "block_id=$PRD_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT - echo "page_id=$PRD_DOCS_PAGE_ID" >> $GITHUB_OUTPUT - fi - id: extract_docs_info - - - name: Generate API docs - env: - DOCS_URL: ${{ secrets.DOCS_URL }} - DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }} - DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }} - DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }} - EVENT: ${{ github.event_name }} - RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }} - run: | - if [ ${EVENT} != "workflow_dispatch" ]; then - sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json - fi - sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json - sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json - sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json - curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL - - - name: Generate External API docs - env: - DOCS_URL: ${{ secrets.DOCS_URL }} - DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }} - DOCS_PAGE_ID: ${{secrets.EXTERNAL_DOCS_PAGE_ID}} - DOCS_BLOCK_ID: ${{secrets.EXTERNAL_DOCS_BLOCK_ID}} - EVENT: ${{ github.event_name }} - RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }} - run: | - if [ ${EVENT} != "workflow_dispatch" ]; then - sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.external.json - fi - sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.external.json - sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.external.json - sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.external.json - curl -X POST -H 'Content-Type: application/json' -d @docsfera.external.json $DOCS_URL + IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }} + run: helmfile -f helmfiles/${ENV}.yaml sync --set image.tag=$IMAGE_TAG diff --git a/build.docker-compose.yml b/build.docker-compose.yml index 7dddc0e..154e7ec 100644 --- a/build.docker-compose.yml +++ b/build.docker-compose.yml @@ -1,4 +1,4 @@ services: maestro: build: . - image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG} + image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd:${IMAGE_TAG} From db2ea09d25c81980c133c35742296379f35c26c1 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 12 Dec 2024 13:54:18 -0300 Subject: [PATCH 07/45] FEAT: create new route to open data iniciative --- docsfera.json | 64 +++++++++++++++++++ package-lock.json | 8 +-- package.json | 2 +- src/app.module.ts | 3 +- src/modules/open-data/dto/create-user.ts | 14 ++++ .../open-data/open-data.controller.spec.ts | 18 ++++++ src/modules/open-data/open-data.controller.ts | 47 ++++++++++++++ src/modules/open-data/open-data.module.ts | 22 +++++++ .../open-data/open-data.service.spec.ts | 18 ++++++ src/modules/open-data/open-data.service.ts | 50 +++++++++++++++ 10 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 src/modules/open-data/dto/create-user.ts create mode 100644 src/modules/open-data/open-data.controller.spec.ts create mode 100644 src/modules/open-data/open-data.controller.ts create mode 100644 src/modules/open-data/open-data.module.ts create mode 100644 src/modules/open-data/open-data.service.spec.ts create mode 100644 src/modules/open-data/open-data.service.ts diff --git a/docsfera.json b/docsfera.json index 381bdfe..0f21c43 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5530,6 +5530,43 @@ "Health" ] } + }, + "/open-data/sharing-ocean-data": { + "post": { + "operationId": "OpenDataController_createUser", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateUserOpenDataDTO" + } + } + } + }, + "responses": { + "201": { + "description": "" + } + }, + "tags": [ + "OpenData" + ] + } } }, "info": { @@ -8377,6 +8414,33 @@ "required": [ "links" ] + }, + "CreateUserOpenDataDTO": { + "type": "object", + "properties": { + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "email": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "enquiryType": { + "type": "string" + } + }, + "required": [ + "firstName", + "lastName", + "email", + "organization", + "enquiryType" + ] } } } diff --git a/package-lock.json b/package-lock.json index ff693d2..c1ea922 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.36.1", + "@dadosfera/protospack-v2": "3.37.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1403,9 +1403,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.36.1", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.36.1.tgz", - "integrity": "sha512-IbkkjoTjhyhglVt4H78n1atNfC3wl5EDDfwhWA9H6VpHLyj8Fbu4XCnZH839oVC5eDtAaq4f4y8RyDueO/L8zg==", + "version": "3.37.0-beta.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.1.tgz", + "integrity": "sha512-AcDxeg7KruBJncOXbLJFHKH/5GQ9VHR8MxU7s1o1lrZVfZY4Ht3UWmvwL1F2b+6DIhTDqzVOsKjfg7KAbuDYuw==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 1daeda0..8300759 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.36.1", + "@dadosfera/protospack-v2": "3.37.0-beta.1", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/app.module.ts b/src/app.module.ts index fa831fd..1cf1611 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -26,9 +26,9 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module'; import { ProductboardModule } from './modules/productboard/productboard.module'; import { MixpanelModule } from './modules/mixpanel/mixpanel.module'; import { CustomersModule } from './modules/customers/customers.module'; +import { OpenDataModule } from './modules/open-data/open-data.module'; @Module({ - controllers: [], providers: [ DadosferaLogger, { @@ -60,6 +60,7 @@ import { CustomersModule } from './modules/customers/customers.module'; CustomersModule, //Always leave HealthModule last, so it is on the bottom of swagger HealthModule, + OpenDataModule, ], }) export class AppModule {} diff --git a/src/modules/open-data/dto/create-user.ts b/src/modules/open-data/dto/create-user.ts new file mode 100644 index 0000000..78a1bfc --- /dev/null +++ b/src/modules/open-data/dto/create-user.ts @@ -0,0 +1,14 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class CreateUserOpenDataDTO { + @ApiProperty() + firstName: string; + @ApiProperty() + lastName: string; + @ApiProperty() + email: string; + @ApiProperty() + organization: string; + @ApiProperty() + enquiryType: string; +} \ No newline at end of file diff --git a/src/modules/open-data/open-data.controller.spec.ts b/src/modules/open-data/open-data.controller.spec.ts new file mode 100644 index 0000000..f9f0ee5 --- /dev/null +++ b/src/modules/open-data/open-data.controller.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { OpenDataController } from './open-data.controller'; + +describe('OpenDataController', () => { + let controller: OpenDataController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [OpenDataController], + }).compile(); + + controller = module.get(OpenDataController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts new file mode 100644 index 0000000..726b0b2 --- /dev/null +++ b/src/modules/open-data/open-data.controller.ts @@ -0,0 +1,47 @@ +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { Body, Controller, Inject, Post, UseFilters } from '@nestjs/common'; +import { ApiCreatedResponse, ApiHeaders, ApiTags } from '@nestjs/swagger'; +import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { UsersService } from '../users/users.service'; +import { Language } from 'src/decorators/language.decorator'; +import { CreateUserOpenDataDTO } from './dto/create-user'; +import { OpenDataService } from './open-data.service'; + +@Controller('open-data') +@ApiInternalOnlyController() +@ApiTags('OpenData') +@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) +@UseFilters(GrpcToHttpExceptionFilter) +export class OpenDataController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private openDataService: OpenDataService, + ) { + this.logger = dadosferaLogger.logger; + } + + @Post("/sharing-ocean-data") + @ApiCreatedResponse() + async createUser( + @Body() body: CreateUserOpenDataDTO + ) { + const OPENDATA_PUBLIC_USERS_GROUP_ID = "401573bb-334f-44b2-b30e-88d4cea31ae9"; + const OPENDATA_CUSTOMER_ID = "f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"; + const roles = [OPENDATA_PUBLIC_USERS_GROUP_ID]; + + this.logger.info('createUser for open data' + JSON.stringify({ body })); + try { + await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, body, roles); + return "success" + } catch (e) { + return e; + } + // return ; + } + +} diff --git a/src/modules/open-data/open-data.module.ts b/src/modules/open-data/open-data.module.ts new file mode 100644 index 0000000..f82b125 --- /dev/null +++ b/src/modules/open-data/open-data.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; +import { OpenDataController } from './open-data.controller'; +import { UsersService } from '../users/users.service'; +import { ClientsModule } from '@nestjs/microservices' +import { DucClient } from '../duc/client.config'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { RolesModule } from '../roles/roles.module'; +import { PermissionsModule } from '../permissions/permissions.module'; +import { OpenDataService } from './open-data.service'; + +const client = new DucClient(); + +@Module({ + controllers: [OpenDataController], + imports: [ + ClientsModule.register([client.providerOptions]), + RolesModule, + // PermissionsModule, + ], + providers: [DadosferaLogger, UsersService, OpenDataService] +}) +export class OpenDataModule {} diff --git a/src/modules/open-data/open-data.service.spec.ts b/src/modules/open-data/open-data.service.spec.ts new file mode 100644 index 0000000..17269fe --- /dev/null +++ b/src/modules/open-data/open-data.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { OpenDataService } from './open-data.service'; + +describe('OpenDataService', () => { + let service: OpenDataService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [OpenDataService], + }).compile(); + + service = module.get(OpenDataService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts new file mode 100644 index 0000000..239f11c --- /dev/null +++ b/src/modules/open-data/open-data.service.ts @@ -0,0 +1,50 @@ +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { lastValueFrom } from 'rxjs'; +import { CreateUserOpenDataDTO } from './dto/create-user'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { DucClient } from '../duc/client.config'; +import { ClientGrpc } from '@nestjs/microservices'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; + +@Injectable() +export class OpenDataService implements OnModuleInit { + logger: DadosferaLogger; + + private usersClientService: UsersProtoService; + constructor( + @Inject(DadosferaLogger) + private dadosferaLogger: DadosferaLogger, + @Inject(DucClient.name) + private readonly grpcClient: ClientGrpc, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.usersClientService = this.grpcClient.getService( + ProtoServices.UsersProtoService, + ); + + } + + async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[]) { + const body = { + email: data.email, + name: data.firstName + " " + data.lastName, + department: data.organization, + jobTitle: data.enquiryType, + customerId: customerId, + roleIds: roleIds + } + + try { + await lastValueFrom( + this.usersClientService.SimpleUserCreate(body), + ); + return "User created"; + } catch(err) { + return err; + } + } +} From 9e2de28032a4c6870741c51863fd5b179ee7158f Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 18 Dec 2024 13:58:08 -0300 Subject: [PATCH 08/45] FIX: map request body to the form wordpress --- src/modules/open-data/dto/create-user.ts | 14 ------ src/modules/open-data/dto/wordpres-form.ts | 48 +++++++++++++++++++ src/modules/open-data/open-data.controller.ts | 35 ++++++++++++-- src/modules/open-data/open-data.service.ts | 2 +- 4 files changed, 79 insertions(+), 20 deletions(-) delete mode 100644 src/modules/open-data/dto/create-user.ts create mode 100644 src/modules/open-data/dto/wordpres-form.ts diff --git a/src/modules/open-data/dto/create-user.ts b/src/modules/open-data/dto/create-user.ts deleted file mode 100644 index 78a1bfc..0000000 --- a/src/modules/open-data/dto/create-user.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { ApiProperty } from "@nestjs/swagger"; - -export class CreateUserOpenDataDTO { - @ApiProperty() - firstName: string; - @ApiProperty() - lastName: string; - @ApiProperty() - email: string; - @ApiProperty() - organization: string; - @ApiProperty() - enquiryType: string; -} \ No newline at end of file diff --git a/src/modules/open-data/dto/wordpres-form.ts b/src/modules/open-data/dto/wordpres-form.ts new file mode 100644 index 0000000..39b6ed7 --- /dev/null +++ b/src/modules/open-data/dto/wordpres-form.ts @@ -0,0 +1,48 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class CreateUserOpenDataDTO { + @ApiProperty() + firstName: string; + @ApiProperty() + lastName: string; + @ApiProperty() + email: string; + @ApiProperty() + organization: string; + @ApiProperty() + enquiryType: string; +} + +type FormField = { + id: string; + type: string; + title: string; + value: string; + raw_value: string; + required: string; +}; + +type MetaData = { + title: string; + value: string; +}; + +export type WordpressForm = { + form: { + id: string; + name: string; + }; + fields: { + [key: string]: FormField; + }; + meta: { + date: MetaData; + time: MetaData; + page_url: MetaData; + user_agent: MetaData; + remote_ip: MetaData; + credit: MetaData; + }; +}; + + diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index 726b0b2..f0354d8 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -6,8 +6,8 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt import { LanguageEnum } from 'src/utils/languages.enum'; import { UsersService } from '../users/users.service'; import { Language } from 'src/decorators/language.decorator'; -import { CreateUserOpenDataDTO } from './dto/create-user'; import { OpenDataService } from './open-data.service'; +import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; @Controller('open-data') @ApiInternalOnlyController() @@ -28,20 +28,45 @@ export class OpenDataController { @Post("/sharing-ocean-data") @ApiCreatedResponse() async createUser( - @Body() body: CreateUserOpenDataDTO + @Body() + body: WordpressForm ) { + this.logger.info('createUser for open data' + JSON.stringify({ body })); + const OPENDATA_PUBLIC_USERS_GROUP_ID = "401573bb-334f-44b2-b30e-88d4cea31ae9"; const OPENDATA_CUSTOMER_ID = "f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"; const roles = [OPENDATA_PUBLIC_USERS_GROUP_ID]; - this.logger.info('createUser for open data' + JSON.stringify({ body })); + const data = {} + try { - await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, body, roles); + Object.keys(body.fields) + .filter(key => body.fields[key].required === "1") + .forEach(key => { + const field = body.fields[key] + data[field.id] = field.value + }); + } catch (e) { + this.logger.error('user data' + e.message); + } + + + const user: CreateUserOpenDataDTO = { + email: data["email"], + enquiryType: data["enquiry_type"], + firstName: data["first_name"], + lastName: data["last_name"], + organization: data["organization"] + } + this.logger.info('user data' + JSON.stringify({ user })); + + try { + await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles); return "success" } catch (e) { + this.logger.error('user data' + e.message); return e; } - // return ; } } diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts index 239f11c..8f37e7f 100644 --- a/src/modules/open-data/open-data.service.ts +++ b/src/modules/open-data/open-data.service.ts @@ -1,6 +1,6 @@ import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; import { lastValueFrom } from 'rxjs'; -import { CreateUserOpenDataDTO } from './dto/create-user'; +import { CreateUserOpenDataDTO } from './dto/wordpres-form'; import DadosferaLogger from '@dadosfera/dadosfera-logs'; import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; import { DucClient } from '../duc/client.config'; From 588808848649e44edda98b7586dd61d473a4bb19 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 18 Dec 2024 14:19:00 -0300 Subject: [PATCH 09/45] FIX: remove unused suite test --- .../open-data/open-data.controller.spec.ts | 18 ------------------ src/modules/open-data/open-data.controller.ts | 6 ++++-- .../open-data/open-data.service.spec.ts | 18 ------------------ 3 files changed, 4 insertions(+), 38 deletions(-) delete mode 100644 src/modules/open-data/open-data.controller.spec.ts delete mode 100644 src/modules/open-data/open-data.service.spec.ts diff --git a/src/modules/open-data/open-data.controller.spec.ts b/src/modules/open-data/open-data.controller.spec.ts deleted file mode 100644 index f9f0ee5..0000000 --- a/src/modules/open-data/open-data.controller.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { OpenDataController } from './open-data.controller'; - -describe('OpenDataController', () => { - let controller: OpenDataController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [OpenDataController], - }).compile(); - - controller = module.get(OpenDataController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index f0354d8..05f915b 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -33,8 +33,10 @@ export class OpenDataController { ) { this.logger.info('createUser for open data' + JSON.stringify({ body })); - const OPENDATA_PUBLIC_USERS_GROUP_ID = "401573bb-334f-44b2-b30e-88d4cea31ae9"; - const OPENDATA_CUSTOMER_ID = "f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"; + // "401573bb-334f-44b2-b30e-88d4cea31ae9" + const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID; + // ""f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"" + const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; const roles = [OPENDATA_PUBLIC_USERS_GROUP_ID]; const data = {} diff --git a/src/modules/open-data/open-data.service.spec.ts b/src/modules/open-data/open-data.service.spec.ts deleted file mode 100644 index 17269fe..0000000 --- a/src/modules/open-data/open-data.service.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Test, TestingModule } from '@nestjs/testing'; -import { OpenDataService } from './open-data.service'; - -describe('OpenDataService', () => { - let service: OpenDataService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [OpenDataService], - }).compile(); - - service = module.get(OpenDataService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); From 2b01b340e4d067dc4ba2be48b291885039cb3834 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 18 Dec 2024 14:19:17 -0300 Subject: [PATCH 10/45] FIX: add env --- environment.d.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.d.ts b/environment.d.ts index 727c484..c910516 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -12,6 +12,8 @@ declare global { INTERNAL_SWAGGER: 'true' | 'false'; AWS_REGION: string; + OPEN_GROUP_ID: string; + OPEN_CUSTOMER_ID: string; } } } From 3026cd07483dab4abc5405301ef9454f41ecd919 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 23 Dec 2024 12:18:42 -0300 Subject: [PATCH 11/45] UPDATE: updating stg endpoints --- helmfiles/stg.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index eb1fc69..95e923d 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -5,12 +5,12 @@ charts: - ../maestro/values.yaml set: - name: maestro.duc_url - value: duc-temp.dadosfera.ai + value: duc.stg.dadosfera.ai - name: hostname - value: maestro-temp.dadosfera.ai + value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory-temp.dadosfera.ai + value: pi-factory.stg.dadosfera.ai - name: maestro.in_factory_url - value: in-factory-temp.dadosfera.ai + value: in-factory.stg.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory-temp.dadosfera.ai \ No newline at end of file + value: in-factory.stg.dadosfera.ai From 3a696ecb0ed9553b54dcc15896f7f69870e0d98e Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 26 Dec 2024 15:31:22 -0300 Subject: [PATCH 12/45] CHORE: update envs --- helmfiles/prd.yaml | 6 +++++- helmfiles/stg.yaml | 6 +++++- maestro/values.yaml | 3 ++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 8076b3b..344cb88 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -13,4 +13,8 @@ charts: - name: maestro.in_factory_url value: in-factory.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.dadosfera.ai \ No newline at end of file + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e + - name: maestro.open_group_id + value: 401573bb-334f-44b2-b30e-88d4cea31ae9 \ No newline at end of file diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index eb1fc69..02315e4 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -13,4 +13,8 @@ charts: - name: maestro.in_factory_url value: in-factory-temp.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory-temp.dadosfera.ai \ No newline at end of file + value: in-factory-temp.dadosfera.ai + - name: maestro.open_customer_id + value: bb4e9b26-d465-40ff-9345-e768ca69a55e + - name: maestro.open_group_id + value: 4d621501-5ea4-444b-b46e-1a009f3132ff \ No newline at end of file diff --git a/maestro/values.yaml b/maestro/values.yaml index a8d745c..070c533 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -40,7 +40,8 @@ maestro: sm_oauth_path: prd/root/oauth_applications tr_factory_url: in-factory.dadosfera.ai upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 - + open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e + open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 autoscaling: enabled: false minReplicas: 1 From 8c122910964ac6e42e6b274354678dd0943b2b4b Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 6 Jan 2025 14:15:29 -0300 Subject: [PATCH 13/45] FIX: try catch in customer controller --- src/modules/customers/customers.controller.ts | 16 ++++++++-- src/modules/customers/customers.service.ts | 29 +++++++------------ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 4268e6a..7afc3fb 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -59,9 +59,21 @@ export class CustomersController { this.logger.info('saveCustomertheme', JSON.stringify({ id, theme: data })); - const theme = await this.customersService.createThemeByCustomer(id, data); - return theme; + try { + // const theme = await this.customersService.createThemeByCustomer(id, data); + this.logger.info('saveCustomertheme', JSON.stringify(data)); + return { theme: null }; + } catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message); + return { theme: null }; + }; + } + } @Get(':id/theme') diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 5dc43f3..4e2ce3a 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -96,32 +96,23 @@ export class CustomersService implements OnModuleInit { throw new HttpException(null, HttpStatus.BAD_REQUEST); } - try { - const result = await firstValueFrom( - this.customerService.CustomerCreateTheme({ - customerId: id, - theme - }), - ); - return { theme: result }; - } catch (err) { - if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { - this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); - throw new HttpException(err.details, HttpStatus.NOT_FOUND); - } else { - this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message); - throw err - }; - } + const result = await firstValueFrom( + this.customerService.CustomerCreateTheme({ + customerId: id, + theme + }), + ); + return { theme: result }; + } - async getThemeByCustomer(id: string): Promise { + async getThemeByCustomer(id: string): Promise { if (!id) { this.logger.error('Error - getCustomerTheme - not found id:' + id); throw new HttpException(null, HttpStatus.BAD_REQUEST); } - const result = await firstValueFrom( + const result = await firstValueFrom( this.customerService.CustomerGetTheme({ id }), From beb086ad3c4ee2a099f847b2f20104c45b337ab9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 6 Jan 2025 14:29:22 -0300 Subject: [PATCH 14/45] FIX: fixed customer and role id --- src/modules/open-data/open-data.controller.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index 05f915b..01f2c1a 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -1,6 +1,6 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; import { Body, Controller, Inject, Post, UseFilters } from '@nestjs/common'; -import { ApiCreatedResponse, ApiHeaders, ApiTags } from '@nestjs/swagger'; +import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; @@ -26,7 +26,7 @@ export class OpenDataController { } @Post("/sharing-ocean-data") - @ApiCreatedResponse() + @ApiOkResponse() async createUser( @Body() body: WordpressForm @@ -34,10 +34,12 @@ export class OpenDataController { this.logger.info('createUser for open data' + JSON.stringify({ body })); // "401573bb-334f-44b2-b30e-88d4cea31ae9" - const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID; + // const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID; // ""f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"" - const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; - const roles = [OPENDATA_PUBLIC_USERS_GROUP_ID]; + // const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; + const OPENDATA_CUSTOMER_ID = "bb4e9b26-d465-40ff-9345-e768ca69a55e"; + + const roles = ["4d621501-5ea4-444b-b46e-1a009f3132ff"]; const data = {} @@ -60,10 +62,11 @@ export class OpenDataController { lastName: data["last_name"], organization: data["organization"] } - this.logger.info('user data' + JSON.stringify({ user })); + this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID })); try { - await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles); + const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles); + this.logger.info('user created with sucessfull data' + JSON.stringify(response)); return "success" } catch (e) { this.logger.error('user data' + e.message); From 7c0b87ffc38dc7ed6e181bcb0ae71027e4ba2312 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 6 Jan 2025 15:25:28 -0300 Subject: [PATCH 15/45] FEAT: decorators to apply an origin in endpoint router --- src/decorators/set-origin.decorator.ts | 32 +++++++++++++++++++ src/modules/open-data/open-data.controller.ts | 5 ++- 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 src/decorators/set-origin.decorator.ts diff --git a/src/decorators/set-origin.decorator.ts b/src/decorators/set-origin.decorator.ts new file mode 100644 index 0000000..c6d5338 --- /dev/null +++ b/src/decorators/set-origin.decorator.ts @@ -0,0 +1,32 @@ +import { Injectable, CanActivate, ExecutionContext, ForbiddenException } from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; +import { SetMetadata } from '@nestjs/common'; + +export const SetOrigin = (origin: string) => SetMetadata('allowedOrigin', origin); + +@Injectable() +export class CORSGuard implements CanActivate { + constructor(private reflector: Reflector) {} + + canActivate(context: ExecutionContext): boolean { + const response = context.switchToHttp().getResponse(); + const request = context.switchToHttp().getRequest(); + const origin = request.headers.origin; + + // Obter a origem permitida através do decorador + const allowedOrigin = this.reflector.get('allowedOrigin', context.getHandler()); + if (process.env.ENV === "local") { + return true; + } + + // Verifica se a origem da requisição é permitida + if (allowedOrigin && origin !== allowedOrigin) { + throw new ForbiddenException('Acesso não permitido pela política CORS'); + } + + response.setHeader('Access-Control-Allow-Origin', allowedOrigin); + + return true; + } +} + diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index 01f2c1a..86252a1 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -1,5 +1,5 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { Body, Controller, Inject, Post, UseFilters } from '@nestjs/common'; +import { Body, Controller, Inject, Post, UseFilters, UseGuards } from '@nestjs/common'; import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @@ -8,6 +8,7 @@ import { UsersService } from '../users/users.service'; import { Language } from 'src/decorators/language.decorator'; import { OpenDataService } from './open-data.service'; import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; +import { CORSGuard, SetOrigin } from 'src/decorators/set-origin.decorator'; @Controller('open-data') @ApiInternalOnlyController() @@ -26,6 +27,8 @@ export class OpenDataController { } @Post("/sharing-ocean-data") + @SetOrigin('https://devsbm.dadosfera.io/') + @UseGuards(CORSGuard) @ApiOkResponse() async createUser( @Body() From 12cc555491bb16d037586ae4ff4ff5e48d702acb Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 6 Jan 2025 16:22:03 -0300 Subject: [PATCH 16/45] FIX: remove origin decorator --- src/modules/open-data/open-data.controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index 86252a1..ce2c3a5 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -27,8 +27,8 @@ export class OpenDataController { } @Post("/sharing-ocean-data") - @SetOrigin('https://devsbm.dadosfera.io/') - @UseGuards(CORSGuard) + // @SetOrigin('https://devsbm.dadosfera.io/') + // @UseGuards(CORSGuard) @ApiOkResponse() async createUser( @Body() From 118c2653a580f8c09f72f30f1a0bc64aa5774e67 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 8 Jan 2025 15:26:24 -0300 Subject: [PATCH 17/45] FEAT: add language by query --- src/modules/open-data/open-data.controller.ts | 13 ++++++++++--- src/modules/open-data/open-data.service.ts | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index ce2c3a5..a20a038 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -1,5 +1,5 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { Body, Controller, Inject, Post, UseFilters, UseGuards } from '@nestjs/common'; +import { Body, Controller, Inject, Param, Post, Query, UseFilters, UseGuards } from '@nestjs/common'; import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @@ -9,6 +9,8 @@ import { Language } from 'src/decorators/language.decorator'; import { OpenDataService } from './open-data.service'; import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; import { CORSGuard, SetOrigin } from 'src/decorators/set-origin.decorator'; +import { Metadata } from '@grpc/grpc-js'; +import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; @Controller('open-data') @ApiInternalOnlyController() @@ -32,7 +34,9 @@ export class OpenDataController { @ApiOkResponse() async createUser( @Body() - body: WordpressForm + body: WordpressForm, + @Query() + language: string ) { this.logger.info('createUser for open data' + JSON.stringify({ body })); @@ -43,6 +47,9 @@ export class OpenDataController { const OPENDATA_CUSTOMER_ID = "bb4e9b26-d465-40ff-9345-e768ca69a55e"; const roles = ["4d621501-5ea4-444b-b46e-1a009f3132ff"]; + const metadata = PackTheMetadata({ + language, + }); const data = {} @@ -68,7 +75,7 @@ export class OpenDataController { this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID })); try { - const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles); + const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); this.logger.info('user created with sucessfull data' + JSON.stringify(response)); return "success" } catch (e) { diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts index 8f37e7f..317b8e3 100644 --- a/src/modules/open-data/open-data.service.ts +++ b/src/modules/open-data/open-data.service.ts @@ -6,6 +6,7 @@ import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfa import { DucClient } from '../duc/client.config'; import { ClientGrpc } from '@nestjs/microservices'; import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { Metadata } from '@grpc/grpc-js'; @Injectable() export class OpenDataService implements OnModuleInit { @@ -28,7 +29,7 @@ export class OpenDataService implements OnModuleInit { } - async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[]) { + async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) { const body = { email: data.email, name: data.firstName + " " + data.lastName, @@ -40,7 +41,7 @@ export class OpenDataService implements OnModuleInit { try { await lastValueFrom( - this.usersClientService.SimpleUserCreate(body), + this.usersClientService.SimpleUserCreate(body, metadata), ); return "User created"; } catch(err) { From 55d9441d10846db955968c722d454d0542ae8de0 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 8 Jan 2025 18:12:04 -0300 Subject: [PATCH 18/45] FIX: remove language param --- src/modules/open-data/open-data.controller.ts | 12 ++++++------ src/modules/open-data/open-data.service.ts | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index a20a038..b2b96f0 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -35,8 +35,8 @@ export class OpenDataController { async createUser( @Body() body: WordpressForm, - @Query() - language: string + // @Query() + // language: string ) { this.logger.info('createUser for open data' + JSON.stringify({ body })); @@ -47,9 +47,9 @@ export class OpenDataController { const OPENDATA_CUSTOMER_ID = "bb4e9b26-d465-40ff-9345-e768ca69a55e"; const roles = ["4d621501-5ea4-444b-b46e-1a009f3132ff"]; - const metadata = PackTheMetadata({ - language, - }); + // const metadata = PackTheMetadata({ + // language: , + // }); const data = {} @@ -75,7 +75,7 @@ export class OpenDataController { this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID })); try { - const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); + const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles); this.logger.info('user created with sucessfull data' + JSON.stringify(response)); return "success" } catch (e) { diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts index 317b8e3..330898d 100644 --- a/src/modules/open-data/open-data.service.ts +++ b/src/modules/open-data/open-data.service.ts @@ -29,7 +29,7 @@ export class OpenDataService implements OnModuleInit { } - async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) { + async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[]) { const body = { email: data.email, name: data.firstName + " " + data.lastName, @@ -41,7 +41,7 @@ export class OpenDataService implements OnModuleInit { try { await lastValueFrom( - this.usersClientService.SimpleUserCreate(body, metadata), + this.usersClientService.SimpleUserCreate(body), ); return "User created"; } catch(err) { From 54d19a78ab41504870c6e2bf05392cc1697542f6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 9 Jan 2025 08:13:06 -0300 Subject: [PATCH 19/45] FIX: param to language and message in json --- src/modules/open-data/open-data.controller.ts | 43 +++++++++++++------ src/modules/open-data/open-data.service.ts | 4 +- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index b2b96f0..c911924 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -1,5 +1,5 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { Body, Controller, Inject, Param, Post, Query, UseFilters, UseGuards } from '@nestjs/common'; +import { Body, Controller, Inject, Param, Post, Query, Req, UseFilters, UseGuards } from '@nestjs/common'; import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @@ -11,6 +11,8 @@ import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; import { CORSGuard, SetOrigin } from 'src/decorators/set-origin.decorator'; import { Metadata } from '@grpc/grpc-js'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { request } from 'http'; +import { Request } from 'express'; @Controller('open-data') @ApiInternalOnlyController() @@ -29,16 +31,23 @@ export class OpenDataController { } @Post("/sharing-ocean-data") - // @SetOrigin('https://devsbm.dadosfera.io/') + // @SetOrigin('devsbm.dadosfera.io') // @UseGuards(CORSGuard) @ApiOkResponse() async createUser( @Body() body: WordpressForm, - // @Query() - // language: string + @Query('language') + language: string, + @Req() + request: Request, ) { - this.logger.info('createUser for open data' + JSON.stringify({ body })); + this.logger.info('createUser for open data' + + JSON.stringify({ + origin: request.headers.origin, + language, + body + })); // "401573bb-334f-44b2-b30e-88d4cea31ae9" // const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID; @@ -47,9 +56,9 @@ export class OpenDataController { const OPENDATA_CUSTOMER_ID = "bb4e9b26-d465-40ff-9345-e768ca69a55e"; const roles = ["4d621501-5ea4-444b-b46e-1a009f3132ff"]; - // const metadata = PackTheMetadata({ - // language: , - // }); + const metadata = PackTheMetadata({ + language: language || 'en-us' + }); const data = {} @@ -59,9 +68,9 @@ export class OpenDataController { .forEach(key => { const field = body.fields[key] data[field.id] = field.value - }); + }); } catch (e) { - this.logger.error('user data' + e.message); + this.logger.error('user data ' + e.message); } @@ -75,13 +84,19 @@ export class OpenDataController { this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID })); try { - const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles); + const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); this.logger.info('user created with sucessfull data' + JSON.stringify(response)); - return "success" + return { + success: true, + message: 'user created with succesfull' + } } catch (e) { this.logger.error('user data' + e.message); - return e; + return { + success: false, + message: e.message + }; } } - + } diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts index 330898d..317b8e3 100644 --- a/src/modules/open-data/open-data.service.ts +++ b/src/modules/open-data/open-data.service.ts @@ -29,7 +29,7 @@ export class OpenDataService implements OnModuleInit { } - async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[]) { + async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) { const body = { email: data.email, name: data.firstName + " " + data.lastName, @@ -41,7 +41,7 @@ export class OpenDataService implements OnModuleInit { try { await lastValueFrom( - this.usersClientService.SimpleUserCreate(body), + this.usersClientService.SimpleUserCreate(body, metadata), ); return "User created"; } catch(err) { From af8fa7237798f4540618bcf94a10906cc19c2eb8 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 9 Jan 2025 08:40:02 -0300 Subject: [PATCH 20/45] FIX: add header content type and change status code to 200 --- src/modules/open-data/open-data.controller.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index c911924..975d0da 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -1,5 +1,5 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { Body, Controller, Inject, Param, Post, Query, Req, UseFilters, UseGuards } from '@nestjs/common'; +import { Body, Controller, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards } from '@nestjs/common'; import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @@ -33,6 +33,8 @@ export class OpenDataController { @Post("/sharing-ocean-data") // @SetOrigin('devsbm.dadosfera.io') // @UseGuards(CORSGuard) + @HttpCode(200) + @Header('content-type', 'application/json') @ApiOkResponse() async createUser( @Body() @@ -84,16 +86,18 @@ export class OpenDataController { this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID })); try { - const response = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); - this.logger.info('user created with sucessfull data' + JSON.stringify(response)); + await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); + this.logger.info('user created with sucessfull data'); return { success: true, + status: 'success', message: 'user created with succesfull' } } catch (e) { - this.logger.error('user data' + e.message); + this.logger.error('failed with exception: ' + e.message); return { success: false, + status: 'failed', message: e.message }; } From 7d0b8755c96dd3674355b2d930fece72125ec472 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 13 Jan 2025 17:35:56 -0300 Subject: [PATCH 21/45] FEAT: image upload in theme --- docsfera.json | 264 ++++++++---------- package-lock.json | 17 +- package.json | 4 +- src/app.module.ts | 6 +- src/modules/customers/customers.controller.ts | 51 +--- src/modules/customers/customers.service.ts | 34 --- src/modules/customers/dtos/customers.ts | 28 +- src/modules/duc/client.config.ts | 2 + src/modules/theme/dtos/customers.ts | 47 ++++ src/modules/theme/theme.controller.ts | 124 ++++++++ src/modules/theme/theme.module.ts | 20 ++ src/modules/theme/theme.service.ts | 133 +++++++++ 12 files changed, 455 insertions(+), 275 deletions(-) create mode 100644 src/modules/theme/dtos/customers.ts create mode 100644 src/modules/theme/theme.controller.ts create mode 100644 src/modules/theme/theme.module.ts create mode 100644 src/modules/theme/theme.service.ts diff --git a/docsfera.json b/docsfera.json index 0f21c43..96f6350 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5371,92 +5371,6 @@ ] } }, - "/customers/{id}/theme": { - "post": { - "operationId": "CustomersController_saveCustomertheme", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerThemeRequest" - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerThemeResponse" - } - } - } - }, - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerThemeResponse" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - }, - "get": { - "operationId": "CustomersController_getCustomerTheme", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerThemeResponse" - } - } - } - } - }, - "tags": [ - "Customers" - ] - } - }, "/customers/token": { "get": { "operationId": "CustomersController_getCustomerToken", @@ -5517,20 +5431,6 @@ ] } }, - "/health": { - "get": { - "operationId": "HealthController_check", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Health" - ] - } - }, "/open-data/sharing-ocean-data": { "post": { "operationId": "OpenDataController_createUser", @@ -5546,6 +5446,37 @@ ], "type": "string" } + }, + { + "name": "language", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "OpenData" + ] + } + }, + "/customers/{id}/theme": { + "post": { + "operationId": "ThemeController_saveCustomertheme", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -5553,18 +5484,78 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CreateUserOpenDataDTO" + "$ref": "#/components/schemas/CustomerThemeRequest" } } } }, "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + }, "201": { "description": "" } }, "tags": [ - "OpenData" + "Theme" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "ThemeController_getCustomerTheme", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + } + }, + "tags": [ + "Theme" + ] + } + }, + "/health": { + "get": { + "operationId": "HealthController_check", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Health" ] } } @@ -8344,27 +8335,37 @@ "links" ] }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, "CustomerThemeRequest": { "type": "object", "properties": { + "displayName": { + "type": "string" + }, "backgroundColor": { "type": "string" }, "textColor": { "type": "string" - }, - "logoWhite": { - "type": "string" - }, - "logoBlack": { - "type": "string" } }, "required": [ + "displayName", "backgroundColor", - "textColor", - "logoWhite", - "logoBlack" + "textColor" ] }, "CustomerTheme": { @@ -8400,47 +8401,6 @@ "required": [ "theme" ] - }, - "CustomerLinkRequest": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CustomerLink" - } - } - }, - "required": [ - "links" - ] - }, - "CreateUserOpenDataDTO": { - "type": "object", - "properties": { - "firstName": { - "type": "string" - }, - "lastName": { - "type": "string" - }, - "email": { - "type": "string" - }, - "organization": { - "type": "string" - }, - "enquiryType": { - "type": "string" - } - }, - "required": [ - "firstName", - "lastName", - "email", - "organization", - "enquiryType" - ] } } } diff --git a/package-lock.json b/package-lock.json index c1ea922..839fb46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.1", + "@dadosfera/protospack-v2": "3.37.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -53,7 +53,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.9", "@types/jwk-to-pem": "^2.0.1", - "@types/multer": "^1.4.7", + "@types/multer": "^1.4.12", "@types/node": "^16.18.52", "@types/passport-facebook": "^2.1.11", "@types/passport-google-oauth20": "^2.0.11", @@ -1403,9 +1403,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.1", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.1.tgz", - "integrity": "sha512-AcDxeg7KruBJncOXbLJFHKH/5GQ9VHR8MxU7s1o1lrZVfZY4Ht3UWmvwL1F2b+6DIhTDqzVOsKjfg7KAbuDYuw==", + "version": "3.37.0-beta.3", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.3.tgz", + "integrity": "sha512-9YSgzkXVqe9QsBxJjfxYxttAiyCXxgq49Xjs3OvnTLCXRDn3XtTlPaC+yHrTEUgCkQyhlDZuswe6f6oBImqIRQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", @@ -3527,10 +3527,11 @@ "dev": true }, "node_modules/@types/multer": { - "version": "1.4.11", - "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.11.tgz", - "integrity": "sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w==", + "version": "1.4.12", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz", + "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==", "dev": true, + "license": "MIT", "dependencies": { "@types/express": "*" } diff --git a/package.json b/package.json index 8300759..41ac8c1 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.1", + "@dadosfera/protospack-v2": "3.37.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -74,7 +74,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.9", "@types/jwk-to-pem": "^2.0.1", - "@types/multer": "^1.4.7", + "@types/multer": "^1.4.12", "@types/node": "^16.18.52", "@types/passport-facebook": "^2.1.11", "@types/passport-google-oauth20": "^2.0.11", diff --git a/src/app.module.ts b/src/app.module.ts index 1cf1611..3e54fde 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -27,6 +27,7 @@ import { ProductboardModule } from './modules/productboard/productboard.module'; import { MixpanelModule } from './modules/mixpanel/mixpanel.module'; import { CustomersModule } from './modules/customers/customers.module'; import { OpenDataModule } from './modules/open-data/open-data.module'; +import { ThemeModule } from './modules/theme/theme.module'; @Module({ providers: [ @@ -58,9 +59,10 @@ import { OpenDataModule } from './modules/open-data/open-data.module'; ProductboardModule, MixpanelModule, CustomersModule, - //Always leave HealthModule last, so it is on the bottom of swagger - HealthModule, OpenDataModule, + ThemeModule, + //Always leave HealthModule last, so it is on the bottom of swagger + HealthModule ], }) export class AppModule {} diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 7afc3fb..1bd9835 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -1,15 +1,12 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { Body, Controller, Get, HttpCode, - HttpException, HttpStatus, Inject, Param, - Post, Put, Query, UseFilters, @@ -22,11 +19,10 @@ import { } from 'src/decorators/authentication.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { CustomersService } from './customers.service'; -import { CustomerLinkRequest, CustomerLinksResponse, CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; +import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers'; import { RequestUser, User } from 'src/decorators/user.decorator'; import type { StringValue } from 'ms'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; -import ErrorCodes from 'src/utils/errorCodes'; @ApiTags('Customers') @Controller('customers') @@ -51,51 +47,6 @@ export class CustomersController { return { links }; } - @Post(':id/theme') - @Authenticated() - @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) - @ApiOkResponse({ type: CustomerThemeResponse }) - async saveCustomertheme(@Param('id') id: string, @Body() data: CustomerThemeRequest) { - this.logger.info('saveCustomertheme', JSON.stringify({ - id, theme: data - })); - - try { - // const theme = await this.customersService.createThemeByCustomer(id, data); - this.logger.info('saveCustomertheme', JSON.stringify(data)); - return { theme: null }; - } catch (err) { - if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { - this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); - throw new HttpException(err.details, HttpStatus.NOT_FOUND); - } else { - this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message); - return { theme: null }; - }; - } - - } - - @Get(':id/theme') - @ApiOkResponse({ type: CustomerThemeResponse }) - async getCustomerTheme(@Param('id') id: string) { - this.logger.info('getCustomerTheme with id' + id); - - try { - const data = await this.customersService.getThemeByCustomer(id); - this.logger.info('Success - getCustomerTheme with id'+ id); - return data; - }catch (err) { - if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { - this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND'); - throw new HttpException(err.details, HttpStatus.NOT_FOUND); - } else { - this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message); - return { theme: null }; - }; - } - } - @Put(':id/links') @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 4e2ce3a..cd7f472 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -28,7 +28,6 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { Metadata } from '@grpc/grpc-js'; import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client'; -import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; import DadosferaLogger from '@dadosfera/dadosfera-logs'; // This function will accept any string, which may result in a bug. @@ -90,39 +89,6 @@ export class CustomersService implements OnModuleInit { } } - async createThemeByCustomer(id: string, theme: CustomerThemeRequest): Promise { - if (!id) { - this.logger.error('Error - saveCustomertheme - not found id:' + id); - throw new HttpException(null, HttpStatus.BAD_REQUEST); - } - - const result = await firstValueFrom( - this.customerService.CustomerCreateTheme({ - customerId: id, - theme - }), - ); - return { theme: result }; - - } - - async getThemeByCustomer(id: string): Promise { - if (!id) { - this.logger.error('Error - getCustomerTheme - not found id:' + id); - throw new HttpException(null, HttpStatus.BAD_REQUEST); - } - - const result = await firstValueFrom( - this.customerService.CustomerGetTheme({ - id - }), - ); - - return { - theme: result - } - } - async generateToken( expiresIn = '30m', data: { customerId: string; userId: string; customerName: string }, diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index e26fc94..db56ee3 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -1,4 +1,4 @@ -import { Link, Theme } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; +import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; export class CustomerLink implements Link { @@ -21,29 +21,3 @@ export class CustomerLinksResponse { links: CustomerLink[]; } -export class CustomerTheme implements Theme { - @ApiProperty() - backgroundColor: string; - @ApiProperty() - textColor: string; - @ApiProperty() - logoWhite: string; - @ApiProperty() - logoBlack: string; -} - -export class CustomerThemeResponse { - @ApiProperty() - theme: CustomerTheme; -} - -export class CustomerThemeRequest { - @ApiProperty() - backgroundColor: string; - @ApiProperty() - textColor: string; - @ApiProperty() - logoWhite: string; - @ApiProperty() - logoBlack: string; -} diff --git a/src/modules/duc/client.config.ts b/src/modules/duc/client.config.ts index 25ce28b..4cf4124 100644 --- a/src/modules/duc/client.config.ts +++ b/src/modules/duc/client.config.ts @@ -29,6 +29,8 @@ export class DucClient { objects: true, arrays: true, }, + maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem + maxReceiveMessageLength: 15 * 1024 * 1024, }, }; diff --git a/src/modules/theme/dtos/customers.ts b/src/modules/theme/dtos/customers.ts new file mode 100644 index 0000000..5d87b8b --- /dev/null +++ b/src/modules/theme/dtos/customers.ts @@ -0,0 +1,47 @@ +import { Link, Theme } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + +export class CustomerLink implements Link { + @ApiProperty() + href: string; + @ApiProperty() + name: string; + @ApiProperty() + description: string; + @ApiPropertyOptional() + iconSrc: string; +} +export class CustomerLinkRequest { + @ApiProperty({ type: [CustomerLink] }) + links: CustomerLink[]; +} + +export class CustomerLinksResponse { + @ApiProperty({ type: [CustomerLink] }) + links: CustomerLink[]; +} + +export class CustomerTheme implements Theme { + @ApiProperty() + backgroundColor: string; + @ApiProperty() + textColor: string; + @ApiProperty() + logoWhite: string; + @ApiProperty() + logoBlack: string; +} + +export class CustomerThemeResponse { + @ApiProperty() + theme: CustomerTheme; +} + +export class CustomerThemeRequest { + @ApiProperty() + displayName: string; + @ApiProperty() + backgroundColor: string; + @ApiProperty() + textColor: string; +} diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts new file mode 100644 index 0000000..f04fb7a --- /dev/null +++ b/src/modules/theme/theme.controller.ts @@ -0,0 +1,124 @@ +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { + Body, + Controller, + Get, + HttpException, + HttpStatus, + Inject, + Param, + Post, + Put, + Query, + UploadedFiles, + UseFilters, + UseInterceptors, +} from '@nestjs/common'; +import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { + Authenticated, + RequireAllPermissions, +} from 'src/decorators/authentication.decorator'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; + +import ErrorCodes from 'src/utils/errorCodes'; +import { AnyFilesInterceptor } from '@nestjs/platform-express'; +import { ThemeService } from './theme.service'; + +@ApiTags('Theme') +@Controller('customers') +@UseFilters(GrpcToHttpExceptionFilter) +export class ThemeController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private themeService: ThemeService, + ) { + this.logger = dadosferaLogger.logger; + } + + + @Post('/:id/theme') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse({ type: CustomerThemeResponse }) + @UseInterceptors(AnyFilesInterceptor()) + async saveCustomertheme( + @Param('id') id: string, + @Body() data: CustomerThemeRequest, + @UploadedFiles() files: Array + ) { + + this.logger.info('saveCustomertheme' + JSON.stringify({ + id, + })); + + const logoWhite = files.find(file => file.fieldname === 'logoWhite'); + const logoBlack = files.find(file => file.fieldname === 'logoBlack'); + console.log(logoWhite) + + this.validFileSize(logoWhite); + this.validFileSize(logoBlack); + this.validMimeType(logoWhite); + this.validMimeType(logoBlack); + + try { + const theme = await this.themeService.createThemeByCustomer(id, { + ...data, + logoWhite, + logoBlack + }); + this.logger.info('saveCustomertheme' + JSON.stringify(theme)); + return { theme: theme }; + } catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message); + return { theme: null }; + }; + } + } + + private validFileSize(file: Express.Multer.File) { + const maxFileSize = 10 * 1024 * 1024; // 10MB + + if (file && file.size > maxFileSize) { + throw new HttpException(`O Arquivo ${file.filename} possui mais de 10MB`, HttpStatus.BAD_REQUEST); + } + } + + private validMimeType(file: Express.Multer.File) { + const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png']; + + if (file && !mimeTypesValid.includes(file.mimetype)) { + throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg ou png`, HttpStatus.BAD_REQUEST); + } + } + + @Get('/:id/theme') + @ApiOkResponse({ type: CustomerThemeResponse }) + async getCustomerTheme(@Param('id') id: string) { + this.logger.info('getCustomerTheme with id' + id); + + try { + const data = await this.themeService.getThemeByCustomer(id); + this.logger.info('Success - getCustomerTheme with id'+ id); + return data; + }catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message); + return { theme: null }; + }; + } + } + +} diff --git a/src/modules/theme/theme.module.ts b/src/modules/theme/theme.module.ts new file mode 100644 index 0000000..4fc3d53 --- /dev/null +++ b/src/modules/theme/theme.module.ts @@ -0,0 +1,20 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { ClientsModule } from '@nestjs/microservices'; +import { ThemeController } from './theme.controller'; +import { ThemeService } from './theme.service'; +import { DucClient } from '../duc/client.config'; + +const ducClient = new DucClient(); + +@Module({ + imports: [ + ClientsModule.register([ + ducClient.providerOptions, + ]), + ], + controllers: [ThemeController], + providers: [ThemeService, DadosferaLogger], + exports: [ThemeService], +}) +export class ThemeModule {} diff --git a/src/modules/theme/theme.service.ts b/src/modules/theme/theme.service.ts new file mode 100644 index 0000000..4c5f33f --- /dev/null +++ b/src/modules/theme/theme.service.ts @@ -0,0 +1,133 @@ +import { + OnModuleInit, + Inject, + Injectable, + HttpException, + HttpStatus, + InternalServerErrorException, +} from '@nestjs/common'; + +import { firstValueFrom, lastValueFrom, ReplaySubject } from 'rxjs'; +import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; +import { DucClient } from '../duc/client.config'; +import { ClientGrpc } from '@nestjs/microservices'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; +import { ThemeProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { ThemeRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; +import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { resolve } from 'path'; +import { Readable } from 'stream'; + +type Files = { + logoWhite: Express.Multer.File, + logoBlack: Express.Multer.File, +} + +// This function will accept any string, which may result in a bug. +@Injectable() +export class ThemeService implements OnModuleInit { + private themeService: ThemeProtoService; + private logger: DadosferaLogger; + + constructor( + @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.themeService = this.grpcClient.getService( + ProtoServices.ThemeProtoService, + ); + } + + async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) { + if (!id) { + this.logger.error('Error - saveCustomertheme - not found id:' + id); + throw new HttpException(null, HttpStatus.BAD_REQUEST); + } + + const customerThemeRequest$ = new ReplaySubject(); + + customerThemeRequest$.next({ + customerId: id, + displayName: theme.displayName, + backgroundColor: theme.backgroundColor, + textColor: theme.textColor, + isMetadata: true, + filename: '', + chunk: Buffer.alloc(0) + }) + + if(theme.logoBlack) { + await this.sendFile(theme.logoBlack, customerThemeRequest$); + } + + if(theme.logoWhite) { + await this.sendFile(theme.logoWhite, customerThemeRequest$); + } + customerThemeRequest$.complete(); + + const stream = this.themeService.CustomerCreateTheme(customerThemeRequest$); + + return lastValueFrom(stream); + } + + async getThemeByCustomer(id: string): Promise { + if (!id) { + this.logger.error('Error - getCustomerTheme - not found id:' + id); + throw new HttpException(null, HttpStatus.BAD_REQUEST); + } + + const { theme } = await firstValueFrom( + this.themeService.CustomerGetTheme({ + id + }), + ); + + return { + theme + } + } + + private async sendFile(file: Express.Multer.File, stream$: ReplaySubject) { + const bufferStream = new Readable({ + highWaterMark: 1024 * 1024, // 1 MB por chunk + read() {} + }); + bufferStream.push(file.buffer); + bufferStream.push(null); + + return new Promise((resolve, reject) => { + bufferStream.on('data', (chunk) => { + console.log('enviando chunk', file.fieldname) + const mimetype = file.mimetype.split('/')[1]; // example image/jpeg + const filename = file.fieldname.concat(".", mimetype); + stream$.next({ + customerId: '', + displayName: '', + backgroundColor: '', + textColor: '', + isMetadata: false, + filename: filename, + chunk: chunk + }); + }); + + bufferStream.on('end', () => { + console.log('terminou de enviar') + resolve(file.filename) + }); + + bufferStream.on('error', (err) => { + console.error('Erro no stream:', err); + reject(err); + }); + }); + } + +} From 38a330e578b5da39a9de1f663de2a9a8570f92f3 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Wed, 15 Jan 2025 10:55:08 -0300 Subject: [PATCH 22/45] FIX: change stg env for prd env --- docsfera.json | 3 --- helmfiles/stg.yaml | 6 +++--- src/modules/theme/theme.controller.ts | 3 ++- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docsfera.json b/docsfera.json index 96f6350..88d9fde 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5499,9 +5499,6 @@ } } } - }, - "201": { - "description": "" } }, "tags": [ diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 21fcf0a..fe6713a 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -9,11 +9,11 @@ charts: - name: hostname value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.stg.dadosfera.ai + value: pi-factory.dadosfera.ai - name: maestro.in_factory_url - value: in-factory.stg.dadosfera.ai + value: in-factory.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.stg.dadosfera.ai + value: in-factory.dadosfera.ai - name: maestro.open_customer_id value: bb4e9b26-d465-40ff-9345-e768ca69a55e - name: maestro.open_group_id diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts index f04fb7a..dba694e 100644 --- a/src/modules/theme/theme.controller.ts +++ b/src/modules/theme/theme.controller.ts @@ -13,6 +13,7 @@ import { UploadedFiles, UseFilters, UseInterceptors, + HttpCode } from '@nestjs/common'; import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; @@ -47,6 +48,7 @@ export class ThemeController { @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse({ type: CustomerThemeResponse }) @UseInterceptors(AnyFilesInterceptor()) + @HttpCode(HttpStatus.OK) async saveCustomertheme( @Param('id') id: string, @Body() data: CustomerThemeRequest, @@ -59,7 +61,6 @@ export class ThemeController { const logoWhite = files.find(file => file.fieldname === 'logoWhite'); const logoBlack = files.find(file => file.fieldname === 'logoBlack'); - console.log(logoWhite) this.validFileSize(logoWhite); this.validFileSize(logoBlack); From 44dcf1b2817c8d8a9b4149250a1a66e4eb1aa2a5 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 16 Jan 2025 18:43:15 -0300 Subject: [PATCH 23/45] FIX: change env for stg --- helmfiles/stg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index fe6713a..e78fb04 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -15,7 +15,7 @@ charts: - name: maestro.tr_factory_url value: in-factory.dadosfera.ai - name: maestro.open_customer_id - value: bb4e9b26-d465-40ff-9345-e768ca69a55e + value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id - value: 4d621501-5ea4-444b-b46e-1a009f3132ff + value: e3f98a2f-7748-4981-8505-7695c8ca8218 From 8237160c6d24b894c962d94eb8a222e5588286c9 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 17 Jan 2025 08:38:03 -0300 Subject: [PATCH 24/45] FIX: env fixed and add logger for the process.env --- src/modules/open-data/open-data.controller.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index 975d0da..4a184b7 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -55,9 +55,11 @@ export class OpenDataController { // const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID; // ""f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"" // const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; - const OPENDATA_CUSTOMER_ID = "bb4e9b26-d465-40ff-9345-e768ca69a55e"; + const OPENDATA_CUSTOMER_ID = "b3e3dfe5-b992-4586-a73c-c0b0c00f615d"; - const roles = ["4d621501-5ea4-444b-b46e-1a009f3132ff"]; + this.logger.info("OPENDATA_CUSTOMER_ID: " + process.env.OPEN_CUSTOMER_ID) + this.logger.info("OPEN_GROUP_ID: " + process.env.OPEN_GROUP_ID) + const roles = ["e3f98a2f-7748-4981-8505-7695c8ca8218"]; const metadata = PackTheMetadata({ language: language || 'en-us' }); From 2a6677e4d9530e6a398ae20d9bc7b9e45d8f4330 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 20 Jan 2025 18:41:01 -0300 Subject: [PATCH 25/45] UPDATE: decrease the number of replicas for stg environment --- helmfiles/stg.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index e78fb04..263570f 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -18,4 +18,5 @@ charts: value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id value: e3f98a2f-7748-4981-8505-7695c8ca8218 - + - name: replicaCount + value: 1 \ No newline at end of file From 3850d4b8ae300622be767068617e08fb6f12cb72 Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 20 Jan 2025 18:47:13 -0300 Subject: [PATCH 26/45] UPDATE: blank commit to force deployment --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 028ca81..bb50d3a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@

+ # Maestro Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços. From e55a1cb6573677faf6cb3ad5e5cb080b9b524f2d Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 20 Jan 2025 18:47:44 -0300 Subject: [PATCH 27/45] CI: removed deploy-k8s workflow --- .github/workflows/deploy-k8s.yml | 73 -------------------------------- 1 file changed, 73 deletions(-) delete mode 100644 .github/workflows/deploy-k8s.yml diff --git a/.github/workflows/deploy-k8s.yml b/.github/workflows/deploy-k8s.yml deleted file mode 100644 index f547ac4..0000000 --- a/.github/workflows/deploy-k8s.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Deploy K8S Modifications - -on: - push: - branches: - - main - - beta - -jobs: - extract_environment: - runs-on: ubuntu-22.04 - outputs: - environment: ${{ steps.extract_environment.outputs.environment }} - steps: - - name: Extract Environment - run: | - if [ ${GITHUB_REF} == "refs/heads/main" ]; then - echo "environment=prd" >> $GITHUB_OUTPUT - elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then - echo "environment=stg" >> $GITHUB_OUTPUT - fi - id: extract_environment - - helmfile-deploy: - needs: [extract_environment] - runs-on: [self-hosted, "prd-azure"] - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Helm - uses: azure/setup-helm@v1 - with: - version: 'v3.9.0' - - - name: Install Azure ClI - run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | bash - - - uses: azure/login@v2 - with: - creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install Helmfile - run: | - wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz - tar -xzf helmfile_0.148.0_linux_amd64.tar.gz - mv helmfile /usr/local/bin/ - helmfile --version - - - name: Install Helm Diff Plugin - run: helm plugin install https://github.com/databus23/helm-diff || true - - - name: Setup kubectl - uses: azure/setup-kubectl@v1 - with: - version: 'v1.30.1' - - - name: Authenticate with cluster - env: - CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} - run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing - - - name: Run Helmfile Apply - env: - ENV: ${{ needs.extract_environment.outputs.environment }} - run: helmfile -f helmfiles/${ENV}.yaml sync From 4f6a9d4b668593c85e8933d0f26e6acf995eab57 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 21 Jan 2025 17:00:34 -0300 Subject: [PATCH 28/45] FEAT: new property in customer obj --- docsfera.json | 6 +++++- src/modules/auth/dtos/login.ts | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docsfera.json b/docsfera.json index 88d9fde..1366d0e 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5619,6 +5619,9 @@ "items": { "type": "string" } + }, + "themeEnabled": { + "type": "boolean" } }, "required": [ @@ -5627,7 +5630,8 @@ "name", "tier", "scheduleLimit", - "links" + "links", + "themeEnabled" ] }, "AuthUser": { diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 33a779b..6e87606 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -77,6 +77,8 @@ export class AuthCustomer { scheduleLimit: string; @ApiProperty() links: Link[]; + @ApiProperty() + themeEnabled: boolean; } export class AuthSignInReq implements AuthSignInRequest { From 8e13541b24374e711850852f466646936e2daf0c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Tue, 21 Jan 2025 17:28:13 -0300 Subject: [PATCH 29/45] FEAT: update protospack --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 839fb46..60dbd2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.3", + "@dadosfera/protospack-v2": "3.37.0-beta.4", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1403,9 +1403,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.3", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.3.tgz", - "integrity": "sha512-9YSgzkXVqe9QsBxJjfxYxttAiyCXxgq49Xjs3OvnTLCXRDn3XtTlPaC+yHrTEUgCkQyhlDZuswe6f6oBImqIRQ==", + "version": "3.37.0-beta.4", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.4.tgz", + "integrity": "sha512-es4OIgv4/X2dYMtveW2Q3/nTOSR7ARnX25241wnuvGEx7XdQGq+ahxHXDG36Yznl+bz51miQHkjpKUw3wut6cQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 41ac8c1..8a2b7c1 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.3", + "@dadosfera/protospack-v2": "3.37.0-beta.4", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 10c4ed179b265a9bc8f16a06592dfc08f07c7a6a Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 22 Jan 2025 11:24:01 -0300 Subject: [PATCH 30/45] UPDATE: add toleration for spot instances --- maestro/templates/deployment.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index 18fad50..f8221d7 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -36,6 +36,11 @@ spec: operator: In values: - backend + tolerations: + - key: "kubernetes.azure.com/scalesetpriority" + operator: "Equal" + value: "spot" + effect: "NoSchedule" containers: - name: maestro From cf99acc6822f7d8f1ccd3b8379129e5fc66244b6 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sun, 26 Jan 2025 16:37:37 -0300 Subject: [PATCH 31/45] FEAT: send images in stream --- src/modules/theme/theme.controller.ts | 2 +- src/modules/theme/theme.service.ts | 38 ++++++++++++++++++++------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts index dba694e..db10ffe 100644 --- a/src/modules/theme/theme.controller.ts +++ b/src/modules/theme/theme.controller.ts @@ -74,7 +74,7 @@ export class ThemeController { logoBlack }); this.logger.info('saveCustomertheme' + JSON.stringify(theme)); - return { theme: theme }; + return theme; } catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); diff --git a/src/modules/theme/theme.service.ts b/src/modules/theme/theme.service.ts index 4c5f33f..b2f4716 100644 --- a/src/modules/theme/theme.service.ts +++ b/src/modules/theme/theme.service.ts @@ -25,7 +25,6 @@ type Files = { logoBlack: Express.Multer.File, } -// This function will accept any string, which may result in a bug. @Injectable() export class ThemeService implements OnModuleInit { private themeService: ThemeProtoService; @@ -95,16 +94,11 @@ export class ThemeService implements OnModuleInit { } private async sendFile(file: Express.Multer.File, stream$: ReplaySubject) { - const bufferStream = new Readable({ - highWaterMark: 1024 * 1024, // 1 MB por chunk - read() {} - }); - bufferStream.push(file.buffer); - bufferStream.push(null); + const chunkSize = 4 * 1024 * 1024; + const bufferStream = new CustomBufferStream(file.buffer, chunkSize); return new Promise((resolve, reject) => { bufferStream.on('data', (chunk) => { - console.log('enviando chunk', file.fieldname) const mimetype = file.mimetype.split('/')[1]; // example image/jpeg const filename = file.fieldname.concat(".", mimetype); stream$.next({ @@ -119,15 +113,39 @@ export class ThemeService implements OnModuleInit { }); bufferStream.on('end', () => { - console.log('terminou de enviar') resolve(file.filename) }); bufferStream.on('error', (err) => { - console.error('Erro no stream:', err); reject(err); }); }); } } + +class CustomBufferStream extends Readable { + buffer: Buffer; + offset: number; + chunkSize: number; + + constructor(buffer: Buffer, chunkSize: number) { + super({ highWaterMark: chunkSize }); // Configura o tamanho do chunk + this.buffer = buffer; + this.offset = 0; + this.chunkSize = chunkSize; + } + + _read() { + if (this.offset < this.buffer.length) { + const end = Math.min(this.offset + this.chunkSize, this.buffer.length); + + const copiedBuf = Uint8Array.prototype.slice.call(this.buffer); + const chunk = copiedBuf.slice(this.offset, end); + this.offset = end; + this.push(chunk); + } else { + this.push(null); + } + } +} From b896b1a9b8284719d20991d3714f4fb06ed231f3 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sun, 26 Jan 2025 16:44:31 -0300 Subject: [PATCH 32/45] REFACTOR: rename props --- docsfera.json | 8 ++++---- package-lock.json | 8 ++++---- package.json | 2 +- src/modules/theme/dtos/customers.ts | 4 ++-- src/modules/theme/theme.controller.ts | 16 ++++++++-------- src/modules/theme/theme.service.ts | 12 ++++++------ 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docsfera.json b/docsfera.json index 1366d0e..33988d7 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8378,18 +8378,18 @@ "textColor": { "type": "string" }, - "logoWhite": { + "logo": { "type": "string" }, - "logoBlack": { + "logoLogin": { "type": "string" } }, "required": [ "backgroundColor", "textColor", - "logoWhite", - "logoBlack" + "logo", + "logoLogin" ] }, "CustomerThemeResponse": { diff --git a/package-lock.json b/package-lock.json index 60dbd2d..84a67fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.4", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1403,9 +1403,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.4", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.4.tgz", - "integrity": "sha512-es4OIgv4/X2dYMtveW2Q3/nTOSR7ARnX25241wnuvGEx7XdQGq+ahxHXDG36Yznl+bz51miQHkjpKUw3wut6cQ==", + "version": "0.0.0", + "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "integrity": "sha512-P/J8rK4PB2sC4XuQ2CWFzEdnEba9XxXZ4JBdsVvLCzaTst5RiSuLPHfQE7VEVC8snEXzhfNgFNGZKQVQqhyfiA==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index 8a2b7c1..c824a10 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.4", + "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/theme/dtos/customers.ts b/src/modules/theme/dtos/customers.ts index 5d87b8b..05f5305 100644 --- a/src/modules/theme/dtos/customers.ts +++ b/src/modules/theme/dtos/customers.ts @@ -27,9 +27,9 @@ export class CustomerTheme implements Theme { @ApiProperty() textColor: string; @ApiProperty() - logoWhite: string; + logo: string; @ApiProperty() - logoBlack: string; + logoLogin: string; } export class CustomerThemeResponse { diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts index db10ffe..94335e7 100644 --- a/src/modules/theme/theme.controller.ts +++ b/src/modules/theme/theme.controller.ts @@ -59,19 +59,19 @@ export class ThemeController { id, })); - const logoWhite = files.find(file => file.fieldname === 'logoWhite'); - const logoBlack = files.find(file => file.fieldname === 'logoBlack'); + const logo = files.find(file => file.fieldname === 'logo'); + const logoLogin = files.find(file => file.fieldname === 'logoLogin'); - this.validFileSize(logoWhite); - this.validFileSize(logoBlack); - this.validMimeType(logoWhite); - this.validMimeType(logoBlack); + this.validFileSize(logo); + this.validFileSize(logoLogin); + this.validMimeType(logo); + this.validMimeType(logoLogin); try { const theme = await this.themeService.createThemeByCustomer(id, { ...data, - logoWhite, - logoBlack + logo, + logoLogin }); this.logger.info('saveCustomertheme' + JSON.stringify(theme)); return theme; diff --git a/src/modules/theme/theme.service.ts b/src/modules/theme/theme.service.ts index b2f4716..64583e7 100644 --- a/src/modules/theme/theme.service.ts +++ b/src/modules/theme/theme.service.ts @@ -21,8 +21,8 @@ import { resolve } from 'path'; import { Readable } from 'stream'; type Files = { - logoWhite: Express.Multer.File, - logoBlack: Express.Multer.File, + logo: Express.Multer.File, + logoLogin: Express.Multer.File, } @Injectable() @@ -62,12 +62,12 @@ export class ThemeService implements OnModuleInit { chunk: Buffer.alloc(0) }) - if(theme.logoBlack) { - await this.sendFile(theme.logoBlack, customerThemeRequest$); + if(theme.logo) { + await this.sendFile(theme.logo, customerThemeRequest$); } - if(theme.logoWhite) { - await this.sendFile(theme.logoWhite, customerThemeRequest$); + if(theme.logoLogin) { + await this.sendFile(theme.logoLogin, customerThemeRequest$); } customerThemeRequest$.complete(); From b5c95874bbec22b39bb331cbe434b23c7721e80f Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 27 Jan 2025 09:14:42 -0300 Subject: [PATCH 33/45] chore: updated package --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 84a67fd..9539649 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1403,9 +1403,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "0.0.0", - "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", - "integrity": "sha512-P/J8rK4PB2sC4XuQ2CWFzEdnEba9XxXZ4JBdsVvLCzaTst5RiSuLPHfQE7VEVC8snEXzhfNgFNGZKQVQqhyfiA==", + "version": "3.37.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.5.tgz", + "integrity": "sha512-dtHMzMU2Qa9c6DpygWr7fib19l1zqbYgBi2wGkiJ6uMAvl1CGu9V0p+JA1FwGTQ0QRy7EkkRVi6X6sH5u/ICpA==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", diff --git a/package.json b/package.json index c824a10..072bfb6 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", From 8e827dc9343dcba10302a39fa5ca09fcfdcb2df4 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 27 Jan 2025 11:47:23 -0300 Subject: [PATCH 34/45] chore: update stg values --- helmfiles/stg.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index e78fb04..b9b933e 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -14,8 +14,8 @@ charts: value: in-factory.dadosfera.ai - name: maestro.tr_factory_url value: in-factory.dadosfera.ai - - name: maestro.open_customer_id + - name: open_customer_id value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - - name: maestro.open_group_id + - name: open_group_id value: e3f98a2f-7748-4981-8505-7695c8ca8218 From e7ad6e5a7b672953c4da4da5eb06b710e02ebf85 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 27 Jan 2025 14:12:38 -0300 Subject: [PATCH 35/45] CHORE: update env --- helmfiles/stg.yaml | 4 ++-- maestro/templates/deployment.yaml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index b9b933e..e78fb04 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -14,8 +14,8 @@ charts: value: in-factory.dadosfera.ai - name: maestro.tr_factory_url value: in-factory.dadosfera.ai - - name: open_customer_id + - name: maestro.open_customer_id value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - - name: open_group_id + - name: maestro.open_group_id value: e3f98a2f-7748-4981-8505-7695c8ca8218 diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index 18fad50..a2577d1 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -90,6 +90,10 @@ spec: value: {{ .Values.maestro.tr_factory_url }} - name: UPLOAD_FILE_AGENT_CONNECTION value: {{ .Values.maestro.upload_file_agent_connection }} + - name: OPEN_CUSTOMER_ID + value: {{ .Values.maestro.open_customer_id }} + - name: OPEN_GROUP_ID + value: {{ .Values.maestro.open_group_id }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: From db78494b1a3df72fcde9745d30bdfeb8ac727a32 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 27 Jan 2025 14:39:12 -0300 Subject: [PATCH 36/45] FIX: test log headers --- src/modules/open-data/open-data.controller.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index 4a184b7..d85c53a 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -46,11 +46,15 @@ export class OpenDataController { ) { this.logger.info('createUser for open data' + JSON.stringify({ - origin: request.headers.origin, language, body })); + this.logger.info('headers' + + JSON.stringify({ + headers: request.headers + })); + // "401573bb-334f-44b2-b30e-88d4cea31ae9" // const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID; // ""f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"" From ea9e727fcf2056d7e2051e33bbfd9507f5dbd5ee Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 30 Jan 2025 15:09:27 -0300 Subject: [PATCH 37/45] FEAT: change dashboard id --- src/modules/pipelinesV2/pipelines.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 9b9be63..f10b8ee 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -328,7 +328,7 @@ export class PipelinesService implements OnModuleInit { const res = await lastValueFrom( this.pipelineReadService.PipelineV2GetDashboardUrl( { - dashboard_id: '83', + dashboard_id: '95', exp: '15m', metabase_customer_name: 'dadosferatech', }, From 867aec092b6672168e7cafbdcf2410940257371c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 30 Jan 2025 16:06:13 -0300 Subject: [PATCH 38/45] FEAT: add svg --- src/modules/theme/theme.controller.ts | 4 ++-- src/modules/theme/theme.service.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts index 94335e7..e3745ce 100644 --- a/src/modules/theme/theme.controller.ts +++ b/src/modules/theme/theme.controller.ts @@ -95,10 +95,10 @@ export class ThemeController { } private validMimeType(file: Express.Multer.File) { - const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png']; + const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png', 'image/svg+xml']; if (file && !mimeTypesValid.includes(file.mimetype)) { - throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg ou png`, HttpStatus.BAD_REQUEST); + throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, png ou svg`, HttpStatus.BAD_REQUEST); } } diff --git a/src/modules/theme/theme.service.ts b/src/modules/theme/theme.service.ts index 64583e7..b62338b 100644 --- a/src/modules/theme/theme.service.ts +++ b/src/modules/theme/theme.service.ts @@ -96,11 +96,18 @@ export class ThemeService implements OnModuleInit { private async sendFile(file: Express.Multer.File, stream$: ReplaySubject) { const chunkSize = 4 * 1024 * 1024; const bufferStream = new CustomBufferStream(file.buffer, chunkSize); + const parseMimitypeForExtension = { + 'image/jpeg': '.jpeg', + 'image/jpg': '.jpg', + 'image/png': '.png', + 'image/svg+xml': '.svg', + } + + const extension = parseMimitypeForExtension[file.mimetype]; return new Promise((resolve, reject) => { bufferStream.on('data', (chunk) => { - const mimetype = file.mimetype.split('/')[1]; // example image/jpeg - const filename = file.fieldname.concat(".", mimetype); + const filename = file.fieldname.concat(extension); stream$.next({ customerId: '', displayName: '', From 1102a6e8e504de1cedcfa667462a33bb81ce7809 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 30 Jan 2025 17:06:09 -0300 Subject: [PATCH 39/45] FEAT: remove personal data in logger and only endpoint for wordpress --- src/decorators/set-origin.decorator.ts | 32 ---------------- src/modules/open-data/open-data.controller.ts | 38 ++++++------------- src/modules/open-data/open-data.service.ts | 4 +- 3 files changed, 14 insertions(+), 60 deletions(-) delete mode 100644 src/decorators/set-origin.decorator.ts diff --git a/src/decorators/set-origin.decorator.ts b/src/decorators/set-origin.decorator.ts deleted file mode 100644 index c6d5338..0000000 --- a/src/decorators/set-origin.decorator.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Injectable, CanActivate, ExecutionContext, ForbiddenException } from '@nestjs/common'; -import { Reflector } from '@nestjs/core'; -import { SetMetadata } from '@nestjs/common'; - -export const SetOrigin = (origin: string) => SetMetadata('allowedOrigin', origin); - -@Injectable() -export class CORSGuard implements CanActivate { - constructor(private reflector: Reflector) {} - - canActivate(context: ExecutionContext): boolean { - const response = context.switchToHttp().getResponse(); - const request = context.switchToHttp().getRequest(); - const origin = request.headers.origin; - - // Obter a origem permitida através do decorador - const allowedOrigin = this.reflector.get('allowedOrigin', context.getHandler()); - if (process.env.ENV === "local") { - return true; - } - - // Verifica se a origem da requisição é permitida - if (allowedOrigin && origin !== allowedOrigin) { - throw new ForbiddenException('Acesso não permitido pela política CORS'); - } - - response.setHeader('Access-Control-Allow-Origin', allowedOrigin); - - return true; - } -} - diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index d85c53a..b999060 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -1,5 +1,5 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { Body, Controller, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards } from '@nestjs/common'; +import { Body, Controller, ForbiddenException, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common'; import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; @@ -8,7 +8,6 @@ import { UsersService } from '../users/users.service'; import { Language } from 'src/decorators/language.decorator'; import { OpenDataService } from './open-data.service'; import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; -import { CORSGuard, SetOrigin } from 'src/decorators/set-origin.decorator'; import { Metadata } from '@grpc/grpc-js'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { request } from 'http'; @@ -31,8 +30,6 @@ export class OpenDataController { } @Post("/sharing-ocean-data") - // @SetOrigin('devsbm.dadosfera.io') - // @UseGuards(CORSGuard) @HttpCode(200) @Header('content-type', 'application/json') @ApiOkResponse() @@ -44,26 +41,16 @@ export class OpenDataController { @Req() request: Request, ) { - this.logger.info('createUser for open data' - + JSON.stringify({ - language, - body - })); + if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) { + throw new ForbiddenException(); + } + this.logger.info('createUser for open data'); - this.logger.info('headers' - + JSON.stringify({ - headers: request.headers - })); + this.logger.info('user-agent', request.headers['user-agent']); - // "401573bb-334f-44b2-b30e-88d4cea31ae9" - // const OPENDATA_PUBLIC_USERS_GROUP_ID = process.env.OPEN_GROUP_ID; - // ""f239718a-a271-4ef9-ae7e-02a2f0f3aa6e"" - // const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; - const OPENDATA_CUSTOMER_ID = "b3e3dfe5-b992-4586-a73c-c0b0c00f615d"; - - this.logger.info("OPENDATA_CUSTOMER_ID: " + process.env.OPEN_CUSTOMER_ID) - this.logger.info("OPEN_GROUP_ID: " + process.env.OPEN_GROUP_ID) - const roles = ["e3f98a2f-7748-4981-8505-7695c8ca8218"]; + const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; + const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID; + const roles = [process.env.OPEN_GROUP_ID]; const metadata = PackTheMetadata({ language: language || 'en-us' }); @@ -81,7 +68,6 @@ export class OpenDataController { this.logger.error('user data ' + e.message); } - const user: CreateUserOpenDataDTO = { email: data["email"], enquiryType: data["enquiry_type"], @@ -89,11 +75,11 @@ export class OpenDataController { lastName: data["last_name"], organization: data["organization"] } - this.logger.info('user request' + JSON.stringify({ user, roles, customer: OPENDATA_CUSTOMER_ID })); + this.logger.info(`user request to group ${OPENDATA_CUSTOMER_ID} with role ${OPENDATA_GROUP_ID}`); try { - await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); - this.logger.info('user created with sucessfull data'); + const id = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); + this.logger.info('user created with id: '+ id); return { success: true, status: 'success', diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts index 317b8e3..4c927fa 100644 --- a/src/modules/open-data/open-data.service.ts +++ b/src/modules/open-data/open-data.service.ts @@ -40,10 +40,10 @@ export class OpenDataService implements OnModuleInit { } try { - await lastValueFrom( + const { user } = await lastValueFrom( this.usersClientService.SimpleUserCreate(body, metadata), ); - return "User created"; + return user.id; } catch(err) { return err; } From bcfef8359ec757a32c4d7a04ae91384f0c183723 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 31 Jan 2025 13:39:15 -0300 Subject: [PATCH 40/45] FIX: return null when not found theme --- src/modules/theme/theme.controller.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts index e3745ce..327c5be 100644 --- a/src/modules/theme/theme.controller.ts +++ b/src/modules/theme/theme.controller.ts @@ -109,8 +109,10 @@ export class ThemeController { try { const data = await this.themeService.getThemeByCustomer(id); - this.logger.info('Success - getCustomerTheme with id'+ id); - return data; + this.logger.info('Success - getCustomerTheme'+ JSON.stringify(data)); + if (data?.theme) return data; + + return { theme: null }; }catch (err) { if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND'); From d3f7664116d9624016592c67aab1e4aed6484f69 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 31 Jan 2025 18:13:33 -0300 Subject: [PATCH 41/45] Revert "Merge pull request #250 from dadosfera/beta" This reverts commit 2ea40004a6d4fe87cd04176be85e1689b0c23da9, reversing changes made to 873f7ea314a6d32cfa72547258613bed1acc508b. --- .github/workflows/deploy-k8s.yml | 73 ++++++++ README.md | 1 - docsfera.json | 172 +----------------- environment.d.ts | 2 - helmfiles/prd.yaml | 6 +- helmfiles/stg.yaml | 16 +- maestro/templates/deployment.yaml | 9 - maestro/values.yaml | 3 +- package-lock.json | 18 +- package.json | 4 +- src/app.module.ts | 7 +- src/modules/auth/dtos/login.ts | 2 - src/modules/customers/customers.controller.ts | 7 +- src/modules/customers/customers.service.ts | 9 +- src/modules/customers/dtos/customers.ts | 1 - src/modules/duc/client.config.ts | 2 - src/modules/open-data/dto/wordpres-form.ts | 48 ----- src/modules/open-data/open-data.controller.ts | 98 ---------- src/modules/open-data/open-data.module.ts | 22 --- src/modules/open-data/open-data.service.ts | 51 ------ src/modules/pipelinesV2/pipelines.service.ts | 2 +- src/modules/theme/dtos/customers.ts | 47 ----- src/modules/theme/theme.controller.ts | 127 ------------- src/modules/theme/theme.module.ts | 20 -- src/modules/theme/theme.service.ts | 158 ---------------- 25 files changed, 100 insertions(+), 805 deletions(-) create mode 100644 .github/workflows/deploy-k8s.yml delete mode 100644 src/modules/open-data/dto/wordpres-form.ts delete mode 100644 src/modules/open-data/open-data.controller.ts delete mode 100644 src/modules/open-data/open-data.module.ts delete mode 100644 src/modules/open-data/open-data.service.ts delete mode 100644 src/modules/theme/dtos/customers.ts delete mode 100644 src/modules/theme/theme.controller.ts delete mode 100644 src/modules/theme/theme.module.ts delete mode 100644 src/modules/theme/theme.service.ts diff --git a/.github/workflows/deploy-k8s.yml b/.github/workflows/deploy-k8s.yml new file mode 100644 index 0000000..f547ac4 --- /dev/null +++ b/.github/workflows/deploy-k8s.yml @@ -0,0 +1,73 @@ +name: Deploy K8S Modifications + +on: + push: + branches: + - main + - beta + +jobs: + extract_environment: + runs-on: ubuntu-22.04 + outputs: + environment: ${{ steps.extract_environment.outputs.environment }} + steps: + - name: Extract Environment + run: | + if [ ${GITHUB_REF} == "refs/heads/main" ]; then + echo "environment=prd" >> $GITHUB_OUTPUT + elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then + echo "environment=stg" >> $GITHUB_OUTPUT + fi + id: extract_environment + + helmfile-deploy: + needs: [extract_environment] + runs-on: [self-hosted, "prd-azure"] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: 'v3.9.0' + + - name: Install Azure ClI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | bash + + - uses: azure/login@v2 + with: + creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + - name: Install Helmfile + run: | + wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz + tar -xzf helmfile_0.148.0_linux_amd64.tar.gz + mv helmfile /usr/local/bin/ + helmfile --version + + - name: Install Helm Diff Plugin + run: helm plugin install https://github.com/databus23/helm-diff || true + + - name: Setup kubectl + uses: azure/setup-kubectl@v1 + with: + version: 'v1.30.1' + + - name: Authenticate with cluster + env: + CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} + run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing + + - name: Run Helmfile Apply + env: + ENV: ${{ needs.extract_environment.outputs.environment }} + run: helmfile -f helmfiles/${ENV}.yaml sync diff --git a/README.md b/README.md index bb50d3a..028ca81 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@

- # Maestro Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços. diff --git a/docsfera.json b/docsfera.json index 33988d7..0922da4 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5431,117 +5431,6 @@ ] } }, - "/open-data/sharing-ocean-data": { - "post": { - "operationId": "OpenDataController_createUser", - "parameters": [ - { - "name": "dadosfera-lang", - "in": "header", - "required": false, - "schema": { - "enum": [ - "pt-br", - "en-us" - ], - "type": "string" - } - }, - { - "name": "language", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "OpenData" - ] - } - }, - "/customers/{id}/theme": { - "post": { - "operationId": "ThemeController_saveCustomertheme", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerThemeRequest" - } - } - } - }, - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerThemeResponse" - } - } - } - } - }, - "tags": [ - "Theme" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - }, - "get": { - "operationId": "ThemeController_getCustomerTheme", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerThemeResponse" - } - } - } - } - }, - "tags": [ - "Theme" - ] - } - }, "/health": { "get": { "operationId": "HealthController_check", @@ -5619,9 +5508,6 @@ "items": { "type": "string" } - }, - "themeEnabled": { - "type": "boolean" } }, "required": [ @@ -5630,8 +5516,7 @@ "name", "tier", "scheduleLimit", - "links", - "themeEnabled" + "links" ] }, "AuthUser": { @@ -8349,61 +8234,8 @@ "required": [ "links" ] - }, - "CustomerThemeRequest": { - "type": "object", - "properties": { - "displayName": { - "type": "string" - }, - "backgroundColor": { - "type": "string" - }, - "textColor": { - "type": "string" - } - }, - "required": [ - "displayName", - "backgroundColor", - "textColor" - ] - }, - "CustomerTheme": { - "type": "object", - "properties": { - "backgroundColor": { - "type": "string" - }, - "textColor": { - "type": "string" - }, - "logo": { - "type": "string" - }, - "logoLogin": { - "type": "string" - } - }, - "required": [ - "backgroundColor", - "textColor", - "logo", - "logoLogin" - ] - }, - "CustomerThemeResponse": { - "type": "object", - "properties": { - "theme": { - "$ref": "#/components/schemas/CustomerTheme" - } - }, - "required": [ - "theme" - ] } } } } -} \ No newline at end of file +} diff --git a/environment.d.ts b/environment.d.ts index c910516..727c484 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -12,8 +12,6 @@ declare global { INTERNAL_SWAGGER: 'true' | 'false'; AWS_REGION: string; - OPEN_GROUP_ID: string; - OPEN_CUSTOMER_ID: string; } } } diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 344cb88..8076b3b 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -13,8 +13,4 @@ charts: - name: maestro.in_factory_url value: in-factory.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.dadosfera.ai - - name: maestro.open_customer_id - value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e - - name: maestro.open_group_id - value: 401573bb-334f-44b2-b30e-88d4cea31ae9 \ No newline at end of file + value: in-factory.dadosfera.ai \ No newline at end of file diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 263570f..eb1fc69 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -5,18 +5,12 @@ charts: - ../maestro/values.yaml set: - name: maestro.duc_url - value: duc.stg.dadosfera.ai + value: duc-temp.dadosfera.ai - name: hostname - value: maestro.stg.dadosfera.ai + value: maestro-temp.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.dadosfera.ai + value: pi-factory-temp.dadosfera.ai - name: maestro.in_factory_url - value: in-factory.dadosfera.ai + value: in-factory-temp.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.dadosfera.ai - - name: maestro.open_customer_id - value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - - name: maestro.open_group_id - value: e3f98a2f-7748-4981-8505-7695c8ca8218 - - name: replicaCount - value: 1 \ No newline at end of file + value: in-factory-temp.dadosfera.ai \ No newline at end of file diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index 74a7a5c..18fad50 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -36,11 +36,6 @@ spec: operator: In values: - backend - tolerations: - - key: "kubernetes.azure.com/scalesetpriority" - operator: "Equal" - value: "spot" - effect: "NoSchedule" containers: - name: maestro @@ -95,10 +90,6 @@ spec: value: {{ .Values.maestro.tr_factory_url }} - name: UPLOAD_FILE_AGENT_CONNECTION value: {{ .Values.maestro.upload_file_agent_connection }} - - name: OPEN_CUSTOMER_ID - value: {{ .Values.maestro.open_customer_id }} - - name: OPEN_GROUP_ID - value: {{ .Values.maestro.open_group_id }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/maestro/values.yaml b/maestro/values.yaml index 070c533..a8d745c 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -40,8 +40,7 @@ maestro: sm_oauth_path: prd/root/oauth_applications tr_factory_url: in-factory.dadosfera.ai upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 - open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e - open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 + autoscaling: enabled: false minReplicas: 1 diff --git a/package-lock.json b/package-lock.json index 9539649..61c7a9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.5", + "@dadosfera/protospack-v2": "3.34.0", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -53,7 +53,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.9", "@types/jwk-to-pem": "^2.0.1", - "@types/multer": "^1.4.12", + "@types/multer": "^1.4.7", "@types/node": "^16.18.52", "@types/passport-facebook": "^2.1.11", "@types/passport-google-oauth20": "^2.0.11", @@ -1403,10 +1403,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-beta.5", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.5.tgz", - "integrity": "sha512-dtHMzMU2Qa9c6DpygWr7fib19l1zqbYgBi2wGkiJ6uMAvl1CGu9V0p+JA1FwGTQ0QRy7EkkRVi6X6sH5u/ICpA==", - "license": "ISC", + "version": "3.34.0", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.34.0.tgz", + "integrity": "sha512-VUjpoHg5/uNNkg1cTxWbUv+20t5CTAf1tB4dhBmGSYjgb+Efo5gi3aO/p3ajYuzOOMOxF+EhjQVouU6BD3E+Xg==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" @@ -3527,11 +3526,10 @@ "dev": true }, "node_modules/@types/multer": { - "version": "1.4.12", - "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz", - "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==", + "version": "1.4.11", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.11.tgz", + "integrity": "sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w==", "dev": true, - "license": "MIT", "dependencies": { "@types/express": "*" } diff --git a/package.json b/package.json index 072bfb6..fbe7eea 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-beta.5", + "@dadosfera/protospack-v2": "3.34.0", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -74,7 +74,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.9", "@types/jwk-to-pem": "^2.0.1", - "@types/multer": "^1.4.12", + "@types/multer": "^1.4.7", "@types/node": "^16.18.52", "@types/passport-facebook": "^2.1.11", "@types/passport-google-oauth20": "^2.0.11", diff --git a/src/app.module.ts b/src/app.module.ts index 3e54fde..fa831fd 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -26,10 +26,9 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module'; import { ProductboardModule } from './modules/productboard/productboard.module'; import { MixpanelModule } from './modules/mixpanel/mixpanel.module'; import { CustomersModule } from './modules/customers/customers.module'; -import { OpenDataModule } from './modules/open-data/open-data.module'; -import { ThemeModule } from './modules/theme/theme.module'; @Module({ + controllers: [], providers: [ DadosferaLogger, { @@ -59,10 +58,8 @@ import { ThemeModule } from './modules/theme/theme.module'; ProductboardModule, MixpanelModule, CustomersModule, - OpenDataModule, - ThemeModule, //Always leave HealthModule last, so it is on the bottom of swagger - HealthModule + HealthModule, ], }) export class AppModule {} diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 6e87606..33a779b 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -77,8 +77,6 @@ export class AuthCustomer { scheduleLimit: string; @ApiProperty() links: Link[]; - @ApiProperty() - themeEnabled: boolean; } export class AuthSignInReq implements AuthSignInRequest { diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 1bd9835..04073ec 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -1,4 +1,5 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { Body, Controller, @@ -17,6 +18,7 @@ import { Authenticated, RequireAllPermissions, } from 'src/decorators/authentication.decorator'; +import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { CustomersService } from './customers.service'; import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers'; @@ -26,6 +28,7 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; @ApiTags('Customers') @Controller('customers') +@Authenticated() @UseFilters(GrpcToHttpExceptionFilter) export class CustomersController { logger: DadosferaLogger; @@ -39,7 +42,6 @@ export class CustomersController { } @Get(':id/links') - @Authenticated() @ApiOkResponse({ type: CustomerLinksResponse }) async getCustomerLinks(@Param('id') id: string) { this.logger.info('getCustomerLinks', { id }); @@ -48,7 +50,6 @@ export class CustomersController { } @Put(':id/links') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse() @HttpCode(HttpStatus.OK) @@ -62,7 +63,6 @@ export class CustomersController { } @Get('token') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN) @ApiProduces('text/plain') async getCustomerToken( @@ -79,7 +79,6 @@ export class CustomersController { } @Get('monitoring-dashboard') - @Authenticated() @RequireAllPermissions( PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD, ) diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index cd7f472..5102db5 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -28,24 +28,19 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { Metadata } from '@grpc/grpc-js'; import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client'; -import DadosferaLogger from '@dadosfera/dadosfera-logs'; // This function will accept any string, which may result in a bug. @Injectable() export class CustomersService implements OnModuleInit { private customerService: CustomersProtoService; - private logger: DadosferaLogger; + private pipelineReadService: ReadService.PipelineV2ReadService; constructor( @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, @Inject(PipelinesClientConfiguration.name) private readonly pipelinesGrpcClient: ClientGrpc, - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - ) { - this.logger = dadosferaLogger.logger; - } + ) {} onModuleInit() { this.customerService = this.grpcClient.getService( diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index db56ee3..c3b3070 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -20,4 +20,3 @@ export class CustomerLinksResponse { @ApiProperty({ type: [CustomerLink] }) links: CustomerLink[]; } - diff --git a/src/modules/duc/client.config.ts b/src/modules/duc/client.config.ts index 4cf4124..25ce28b 100644 --- a/src/modules/duc/client.config.ts +++ b/src/modules/duc/client.config.ts @@ -29,8 +29,6 @@ export class DucClient { objects: true, arrays: true, }, - maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem - maxReceiveMessageLength: 15 * 1024 * 1024, }, }; diff --git a/src/modules/open-data/dto/wordpres-form.ts b/src/modules/open-data/dto/wordpres-form.ts deleted file mode 100644 index 39b6ed7..0000000 --- a/src/modules/open-data/dto/wordpres-form.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { ApiProperty } from "@nestjs/swagger"; - -export class CreateUserOpenDataDTO { - @ApiProperty() - firstName: string; - @ApiProperty() - lastName: string; - @ApiProperty() - email: string; - @ApiProperty() - organization: string; - @ApiProperty() - enquiryType: string; -} - -type FormField = { - id: string; - type: string; - title: string; - value: string; - raw_value: string; - required: string; -}; - -type MetaData = { - title: string; - value: string; -}; - -export type WordpressForm = { - form: { - id: string; - name: string; - }; - fields: { - [key: string]: FormField; - }; - meta: { - date: MetaData; - time: MetaData; - page_url: MetaData; - user_agent: MetaData; - remote_ip: MetaData; - credit: MetaData; - }; -}; - - diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts deleted file mode 100644 index b999060..0000000 --- a/src/modules/open-data/open-data.controller.ts +++ /dev/null @@ -1,98 +0,0 @@ -import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { Body, Controller, ForbiddenException, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common'; -import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; -import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; -import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; -import { LanguageEnum } from 'src/utils/languages.enum'; -import { UsersService } from '../users/users.service'; -import { Language } from 'src/decorators/language.decorator'; -import { OpenDataService } from './open-data.service'; -import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; -import { Metadata } from '@grpc/grpc-js'; -import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; -import { request } from 'http'; -import { Request } from 'express'; - -@Controller('open-data') -@ApiInternalOnlyController() -@ApiTags('OpenData') -@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) -@UseFilters(GrpcToHttpExceptionFilter) -export class OpenDataController { - logger: DadosferaLogger; - - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private openDataService: OpenDataService, - ) { - this.logger = dadosferaLogger.logger; - } - - @Post("/sharing-ocean-data") - @HttpCode(200) - @Header('content-type', 'application/json') - @ApiOkResponse() - async createUser( - @Body() - body: WordpressForm, - @Query('language') - language: string, - @Req() - request: Request, - ) { - if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) { - throw new ForbiddenException(); - } - this.logger.info('createUser for open data'); - - this.logger.info('user-agent', request.headers['user-agent']); - - const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; - const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID; - const roles = [process.env.OPEN_GROUP_ID]; - const metadata = PackTheMetadata({ - language: language || 'en-us' - }); - - const data = {} - - try { - Object.keys(body.fields) - .filter(key => body.fields[key].required === "1") - .forEach(key => { - const field = body.fields[key] - data[field.id] = field.value - }); - } catch (e) { - this.logger.error('user data ' + e.message); - } - - const user: CreateUserOpenDataDTO = { - email: data["email"], - enquiryType: data["enquiry_type"], - firstName: data["first_name"], - lastName: data["last_name"], - organization: data["organization"] - } - this.logger.info(`user request to group ${OPENDATA_CUSTOMER_ID} with role ${OPENDATA_GROUP_ID}`); - - try { - const id = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); - this.logger.info('user created with id: '+ id); - return { - success: true, - status: 'success', - message: 'user created with succesfull' - } - } catch (e) { - this.logger.error('failed with exception: ' + e.message); - return { - success: false, - status: 'failed', - message: e.message - }; - } - } - -} diff --git a/src/modules/open-data/open-data.module.ts b/src/modules/open-data/open-data.module.ts deleted file mode 100644 index f82b125..0000000 --- a/src/modules/open-data/open-data.module.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Module } from '@nestjs/common'; -import { OpenDataController } from './open-data.controller'; -import { UsersService } from '../users/users.service'; -import { ClientsModule } from '@nestjs/microservices' -import { DucClient } from '../duc/client.config'; -import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { RolesModule } from '../roles/roles.module'; -import { PermissionsModule } from '../permissions/permissions.module'; -import { OpenDataService } from './open-data.service'; - -const client = new DucClient(); - -@Module({ - controllers: [OpenDataController], - imports: [ - ClientsModule.register([client.providerOptions]), - RolesModule, - // PermissionsModule, - ], - providers: [DadosferaLogger, UsersService, OpenDataService] -}) -export class OpenDataModule {} diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts deleted file mode 100644 index 4c927fa..0000000 --- a/src/modules/open-data/open-data.service.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; -import { lastValueFrom } from 'rxjs'; -import { CreateUserOpenDataDTO } from './dto/wordpres-form'; -import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; -import { DucClient } from '../duc/client.config'; -import { ClientGrpc } from '@nestjs/microservices'; -import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; -import { Metadata } from '@grpc/grpc-js'; - -@Injectable() -export class OpenDataService implements OnModuleInit { - logger: DadosferaLogger; - - private usersClientService: UsersProtoService; - constructor( - @Inject(DadosferaLogger) - private dadosferaLogger: DadosferaLogger, - @Inject(DucClient.name) - private readonly grpcClient: ClientGrpc, - ) { - this.logger = dadosferaLogger.logger; - } - - onModuleInit() { - this.usersClientService = this.grpcClient.getService( - ProtoServices.UsersProtoService, - ); - - } - - async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) { - const body = { - email: data.email, - name: data.firstName + " " + data.lastName, - department: data.organization, - jobTitle: data.enquiryType, - customerId: customerId, - roleIds: roleIds - } - - try { - const { user } = await lastValueFrom( - this.usersClientService.SimpleUserCreate(body, metadata), - ); - return user.id; - } catch(err) { - return err; - } - } -} diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index f10b8ee..9b9be63 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -328,7 +328,7 @@ export class PipelinesService implements OnModuleInit { const res = await lastValueFrom( this.pipelineReadService.PipelineV2GetDashboardUrl( { - dashboard_id: '95', + dashboard_id: '83', exp: '15m', metabase_customer_name: 'dadosferatech', }, diff --git a/src/modules/theme/dtos/customers.ts b/src/modules/theme/dtos/customers.ts deleted file mode 100644 index 05f5305..0000000 --- a/src/modules/theme/dtos/customers.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Link, Theme } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; -import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; - -export class CustomerLink implements Link { - @ApiProperty() - href: string; - @ApiProperty() - name: string; - @ApiProperty() - description: string; - @ApiPropertyOptional() - iconSrc: string; -} -export class CustomerLinkRequest { - @ApiProperty({ type: [CustomerLink] }) - links: CustomerLink[]; -} - -export class CustomerLinksResponse { - @ApiProperty({ type: [CustomerLink] }) - links: CustomerLink[]; -} - -export class CustomerTheme implements Theme { - @ApiProperty() - backgroundColor: string; - @ApiProperty() - textColor: string; - @ApiProperty() - logo: string; - @ApiProperty() - logoLogin: string; -} - -export class CustomerThemeResponse { - @ApiProperty() - theme: CustomerTheme; -} - -export class CustomerThemeRequest { - @ApiProperty() - displayName: string; - @ApiProperty() - backgroundColor: string; - @ApiProperty() - textColor: string; -} diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts deleted file mode 100644 index 327c5be..0000000 --- a/src/modules/theme/theme.controller.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { - Body, - Controller, - Get, - HttpException, - HttpStatus, - Inject, - Param, - Post, - Put, - Query, - UploadedFiles, - UseFilters, - UseInterceptors, - HttpCode -} from '@nestjs/common'; -import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; -import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; -import { - Authenticated, - RequireAllPermissions, -} from 'src/decorators/authentication.decorator'; -import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; -import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; - -import ErrorCodes from 'src/utils/errorCodes'; -import { AnyFilesInterceptor } from '@nestjs/platform-express'; -import { ThemeService } from './theme.service'; - -@ApiTags('Theme') -@Controller('customers') -@UseFilters(GrpcToHttpExceptionFilter) -export class ThemeController { - logger: DadosferaLogger; - - constructor( - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - private themeService: ThemeService, - ) { - this.logger = dadosferaLogger.logger; - } - - - @Post('/:id/theme') - @Authenticated() - @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) - @ApiOkResponse({ type: CustomerThemeResponse }) - @UseInterceptors(AnyFilesInterceptor()) - @HttpCode(HttpStatus.OK) - async saveCustomertheme( - @Param('id') id: string, - @Body() data: CustomerThemeRequest, - @UploadedFiles() files: Array - ) { - - this.logger.info('saveCustomertheme' + JSON.stringify({ - id, - })); - - const logo = files.find(file => file.fieldname === 'logo'); - const logoLogin = files.find(file => file.fieldname === 'logoLogin'); - - this.validFileSize(logo); - this.validFileSize(logoLogin); - this.validMimeType(logo); - this.validMimeType(logoLogin); - - try { - const theme = await this.themeService.createThemeByCustomer(id, { - ...data, - logo, - logoLogin - }); - this.logger.info('saveCustomertheme' + JSON.stringify(theme)); - return theme; - } catch (err) { - if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { - this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); - throw new HttpException(err.details, HttpStatus.NOT_FOUND); - } else { - this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message); - return { theme: null }; - }; - } - } - - private validFileSize(file: Express.Multer.File) { - const maxFileSize = 10 * 1024 * 1024; // 10MB - - if (file && file.size > maxFileSize) { - throw new HttpException(`O Arquivo ${file.filename} possui mais de 10MB`, HttpStatus.BAD_REQUEST); - } - } - - private validMimeType(file: Express.Multer.File) { - const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png', 'image/svg+xml']; - - if (file && !mimeTypesValid.includes(file.mimetype)) { - throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, png ou svg`, HttpStatus.BAD_REQUEST); - } - } - - @Get('/:id/theme') - @ApiOkResponse({ type: CustomerThemeResponse }) - async getCustomerTheme(@Param('id') id: string) { - this.logger.info('getCustomerTheme with id' + id); - - try { - const data = await this.themeService.getThemeByCustomer(id); - this.logger.info('Success - getCustomerTheme'+ JSON.stringify(data)); - if (data?.theme) return data; - - return { theme: null }; - }catch (err) { - if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { - this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND'); - throw new HttpException(err.details, HttpStatus.NOT_FOUND); - } else { - this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message); - return { theme: null }; - }; - } - } - -} diff --git a/src/modules/theme/theme.module.ts b/src/modules/theme/theme.module.ts deleted file mode 100644 index 4fc3d53..0000000 --- a/src/modules/theme/theme.module.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Module } from '@nestjs/common'; -import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { ClientsModule } from '@nestjs/microservices'; -import { ThemeController } from './theme.controller'; -import { ThemeService } from './theme.service'; -import { DucClient } from '../duc/client.config'; - -const ducClient = new DucClient(); - -@Module({ - imports: [ - ClientsModule.register([ - ducClient.providerOptions, - ]), - ], - controllers: [ThemeController], - providers: [ThemeService, DadosferaLogger], - exports: [ThemeService], -}) -export class ThemeModule {} diff --git a/src/modules/theme/theme.service.ts b/src/modules/theme/theme.service.ts deleted file mode 100644 index b62338b..0000000 --- a/src/modules/theme/theme.service.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { - OnModuleInit, - Inject, - Injectable, - HttpException, - HttpStatus, - InternalServerErrorException, -} from '@nestjs/common'; - -import { firstValueFrom, lastValueFrom, ReplaySubject } from 'rxjs'; -import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; -import { DucClient } from '../duc/client.config'; -import { ClientGrpc } from '@nestjs/microservices'; -import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; -import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; -import { ThemeProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; -import { ThemeRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; -import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; -import DadosferaLogger from '@dadosfera/dadosfera-logs'; -import { resolve } from 'path'; -import { Readable } from 'stream'; - -type Files = { - logo: Express.Multer.File, - logoLogin: Express.Multer.File, -} - -@Injectable() -export class ThemeService implements OnModuleInit { - private themeService: ThemeProtoService; - private logger: DadosferaLogger; - - constructor( - @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, - @Inject(DadosferaLogger) - dadosferaLogger: DadosferaLogger, - ) { - this.logger = dadosferaLogger.logger; - } - - onModuleInit() { - this.themeService = this.grpcClient.getService( - ProtoServices.ThemeProtoService, - ); - } - - async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) { - if (!id) { - this.logger.error('Error - saveCustomertheme - not found id:' + id); - throw new HttpException(null, HttpStatus.BAD_REQUEST); - } - - const customerThemeRequest$ = new ReplaySubject(); - - customerThemeRequest$.next({ - customerId: id, - displayName: theme.displayName, - backgroundColor: theme.backgroundColor, - textColor: theme.textColor, - isMetadata: true, - filename: '', - chunk: Buffer.alloc(0) - }) - - if(theme.logo) { - await this.sendFile(theme.logo, customerThemeRequest$); - } - - if(theme.logoLogin) { - await this.sendFile(theme.logoLogin, customerThemeRequest$); - } - customerThemeRequest$.complete(); - - const stream = this.themeService.CustomerCreateTheme(customerThemeRequest$); - - return lastValueFrom(stream); - } - - async getThemeByCustomer(id: string): Promise { - if (!id) { - this.logger.error('Error - getCustomerTheme - not found id:' + id); - throw new HttpException(null, HttpStatus.BAD_REQUEST); - } - - const { theme } = await firstValueFrom( - this.themeService.CustomerGetTheme({ - id - }), - ); - - return { - theme - } - } - - private async sendFile(file: Express.Multer.File, stream$: ReplaySubject) { - const chunkSize = 4 * 1024 * 1024; - const bufferStream = new CustomBufferStream(file.buffer, chunkSize); - const parseMimitypeForExtension = { - 'image/jpeg': '.jpeg', - 'image/jpg': '.jpg', - 'image/png': '.png', - 'image/svg+xml': '.svg', - } - - const extension = parseMimitypeForExtension[file.mimetype]; - - return new Promise((resolve, reject) => { - bufferStream.on('data', (chunk) => { - const filename = file.fieldname.concat(extension); - stream$.next({ - customerId: '', - displayName: '', - backgroundColor: '', - textColor: '', - isMetadata: false, - filename: filename, - chunk: chunk - }); - }); - - bufferStream.on('end', () => { - resolve(file.filename) - }); - - bufferStream.on('error', (err) => { - reject(err); - }); - }); - } - -} - -class CustomBufferStream extends Readable { - buffer: Buffer; - offset: number; - chunkSize: number; - - constructor(buffer: Buffer, chunkSize: number) { - super({ highWaterMark: chunkSize }); // Configura o tamanho do chunk - this.buffer = buffer; - this.offset = 0; - this.chunkSize = chunkSize; - } - - _read() { - if (this.offset < this.buffer.length) { - const end = Math.min(this.offset + this.chunkSize, this.buffer.length); - - const copiedBuf = Uint8Array.prototype.slice.call(this.buffer); - const chunk = copiedBuf.slice(this.offset, end); - this.offset = end; - this.push(chunk); - } else { - this.push(null); - } - } -} From 132614b551519cb0c62b7e0cd0163f39b365ad2c Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Fri, 31 Jan 2025 18:24:08 -0300 Subject: [PATCH 42/45] FEAT: update monitoring dash --- src/modules/pipelinesV2/pipelines.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index 9b9be63..f10b8ee 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -328,7 +328,7 @@ export class PipelinesService implements OnModuleInit { const res = await lastValueFrom( this.pipelineReadService.PipelineV2GetDashboardUrl( { - dashboard_id: '83', + dashboard_id: '95', exp: '15m', metabase_customer_name: 'dadosferatech', }, From d86a011bfdaf7ccdee539c48ef158fe01efad078 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Sat, 1 Feb 2025 16:49:18 -0300 Subject: [PATCH 43/45] Reapply "Merge pull request #250 from dadosfera/beta" This reverts commit d3f7664116d9624016592c67aab1e4aed6484f69. --- .github/workflows/deploy-k8s.yml | 73 -------- README.md | 1 + docsfera.json | 172 +++++++++++++++++- environment.d.ts | 2 + helmfiles/prd.yaml | 6 +- helmfiles/stg.yaml | 16 +- maestro/templates/deployment.yaml | 9 + maestro/values.yaml | 3 +- package-lock.json | 18 +- package.json | 4 +- src/app.module.ts | 7 +- src/modules/auth/dtos/login.ts | 2 + src/modules/customers/customers.controller.ts | 7 +- src/modules/customers/customers.service.ts | 9 +- src/modules/customers/dtos/customers.ts | 1 + src/modules/duc/client.config.ts | 2 + src/modules/open-data/dto/wordpres-form.ts | 48 +++++ src/modules/open-data/open-data.controller.ts | 98 ++++++++++ src/modules/open-data/open-data.module.ts | 22 +++ src/modules/open-data/open-data.service.ts | 51 ++++++ src/modules/theme/dtos/customers.ts | 47 +++++ src/modules/theme/theme.controller.ts | 127 +++++++++++++ src/modules/theme/theme.module.ts | 20 ++ src/modules/theme/theme.service.ts | 158 ++++++++++++++++ 24 files changed, 804 insertions(+), 99 deletions(-) delete mode 100644 .github/workflows/deploy-k8s.yml create mode 100644 src/modules/open-data/dto/wordpres-form.ts create mode 100644 src/modules/open-data/open-data.controller.ts create mode 100644 src/modules/open-data/open-data.module.ts create mode 100644 src/modules/open-data/open-data.service.ts create mode 100644 src/modules/theme/dtos/customers.ts create mode 100644 src/modules/theme/theme.controller.ts create mode 100644 src/modules/theme/theme.module.ts create mode 100644 src/modules/theme/theme.service.ts diff --git a/.github/workflows/deploy-k8s.yml b/.github/workflows/deploy-k8s.yml deleted file mode 100644 index f547ac4..0000000 --- a/.github/workflows/deploy-k8s.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Deploy K8S Modifications - -on: - push: - branches: - - main - - beta - -jobs: - extract_environment: - runs-on: ubuntu-22.04 - outputs: - environment: ${{ steps.extract_environment.outputs.environment }} - steps: - - name: Extract Environment - run: | - if [ ${GITHUB_REF} == "refs/heads/main" ]; then - echo "environment=prd" >> $GITHUB_OUTPUT - elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then - echo "environment=stg" >> $GITHUB_OUTPUT - fi - id: extract_environment - - helmfile-deploy: - needs: [extract_environment] - runs-on: [self-hosted, "prd-azure"] - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up Helm - uses: azure/setup-helm@v1 - with: - version: 'v3.9.0' - - - name: Install Azure ClI - run: | - curl -sL https://aka.ms/InstallAzureCLIDeb | bash - - - uses: azure/login@v2 - with: - creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - - name: Install Helmfile - run: | - wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz - tar -xzf helmfile_0.148.0_linux_amd64.tar.gz - mv helmfile /usr/local/bin/ - helmfile --version - - - name: Install Helm Diff Plugin - run: helm plugin install https://github.com/databus23/helm-diff || true - - - name: Setup kubectl - uses: azure/setup-kubectl@v1 - with: - version: 'v1.30.1' - - - name: Authenticate with cluster - env: - CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }} - run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing - - - name: Run Helmfile Apply - env: - ENV: ${{ needs.extract_environment.outputs.environment }} - run: helmfile -f helmfiles/${ENV}.yaml sync diff --git a/README.md b/README.md index 028ca81..bb50d3a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@

+ # Maestro Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços. diff --git a/docsfera.json b/docsfera.json index 0922da4..33988d7 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5431,6 +5431,117 @@ ] } }, + "/open-data/sharing-ocean-data": { + "post": { + "operationId": "OpenDataController_createUser", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "language", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "OpenData" + ] + } + }, + "/customers/{id}/theme": { + "post": { + "operationId": "ThemeController_saveCustomertheme", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeRequest" + } + } + } + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + } + }, + "tags": [ + "Theme" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "ThemeController_getCustomerTheme", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerThemeResponse" + } + } + } + } + }, + "tags": [ + "Theme" + ] + } + }, "/health": { "get": { "operationId": "HealthController_check", @@ -5508,6 +5619,9 @@ "items": { "type": "string" } + }, + "themeEnabled": { + "type": "boolean" } }, "required": [ @@ -5516,7 +5630,8 @@ "name", "tier", "scheduleLimit", - "links" + "links", + "themeEnabled" ] }, "AuthUser": { @@ -8234,8 +8349,61 @@ "required": [ "links" ] + }, + "CustomerThemeRequest": { + "type": "object", + "properties": { + "displayName": { + "type": "string" + }, + "backgroundColor": { + "type": "string" + }, + "textColor": { + "type": "string" + } + }, + "required": [ + "displayName", + "backgroundColor", + "textColor" + ] + }, + "CustomerTheme": { + "type": "object", + "properties": { + "backgroundColor": { + "type": "string" + }, + "textColor": { + "type": "string" + }, + "logo": { + "type": "string" + }, + "logoLogin": { + "type": "string" + } + }, + "required": [ + "backgroundColor", + "textColor", + "logo", + "logoLogin" + ] + }, + "CustomerThemeResponse": { + "type": "object", + "properties": { + "theme": { + "$ref": "#/components/schemas/CustomerTheme" + } + }, + "required": [ + "theme" + ] } } } } -} +} \ No newline at end of file diff --git a/environment.d.ts b/environment.d.ts index 727c484..c910516 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -12,6 +12,8 @@ declare global { INTERNAL_SWAGGER: 'true' | 'false'; AWS_REGION: string; + OPEN_GROUP_ID: string; + OPEN_CUSTOMER_ID: string; } } } diff --git a/helmfiles/prd.yaml b/helmfiles/prd.yaml index 8076b3b..344cb88 100644 --- a/helmfiles/prd.yaml +++ b/helmfiles/prd.yaml @@ -13,4 +13,8 @@ charts: - name: maestro.in_factory_url value: in-factory.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.dadosfera.ai \ No newline at end of file + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e + - name: maestro.open_group_id + value: 401573bb-334f-44b2-b30e-88d4cea31ae9 \ No newline at end of file diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index eb1fc69..263570f 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -5,12 +5,18 @@ charts: - ../maestro/values.yaml set: - name: maestro.duc_url - value: duc-temp.dadosfera.ai + value: duc.stg.dadosfera.ai - name: hostname - value: maestro-temp.dadosfera.ai + value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory-temp.dadosfera.ai + value: pi-factory.dadosfera.ai - name: maestro.in_factory_url - value: in-factory-temp.dadosfera.ai + value: in-factory.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory-temp.dadosfera.ai \ No newline at end of file + value: in-factory.dadosfera.ai + - name: maestro.open_customer_id + value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d + - name: maestro.open_group_id + value: e3f98a2f-7748-4981-8505-7695c8ca8218 + - name: replicaCount + value: 1 \ No newline at end of file diff --git a/maestro/templates/deployment.yaml b/maestro/templates/deployment.yaml index 18fad50..74a7a5c 100644 --- a/maestro/templates/deployment.yaml +++ b/maestro/templates/deployment.yaml @@ -36,6 +36,11 @@ spec: operator: In values: - backend + tolerations: + - key: "kubernetes.azure.com/scalesetpriority" + operator: "Equal" + value: "spot" + effect: "NoSchedule" containers: - name: maestro @@ -90,6 +95,10 @@ spec: value: {{ .Values.maestro.tr_factory_url }} - name: UPLOAD_FILE_AGENT_CONNECTION value: {{ .Values.maestro.upload_file_agent_connection }} + - name: OPEN_CUSTOMER_ID + value: {{ .Values.maestro.open_customer_id }} + - name: OPEN_GROUP_ID + value: {{ .Values.maestro.open_group_id }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/maestro/values.yaml b/maestro/values.yaml index a8d745c..070c533 100644 --- a/maestro/values.yaml +++ b/maestro/values.yaml @@ -40,7 +40,8 @@ maestro: sm_oauth_path: prd/root/oauth_applications tr_factory_url: in-factory.dadosfera.ai upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911 - + open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e + open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 autoscaling: enabled: false minReplicas: 1 diff --git a/package-lock.json b/package-lock.json index 61c7a9a..9539649 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.34.0", + "@dadosfera/protospack-v2": "3.37.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -53,7 +53,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.9", "@types/jwk-to-pem": "^2.0.1", - "@types/multer": "^1.4.7", + "@types/multer": "^1.4.12", "@types/node": "^16.18.52", "@types/passport-facebook": "^2.1.11", "@types/passport-google-oauth20": "^2.0.11", @@ -1403,9 +1403,10 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.34.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.34.0.tgz", - "integrity": "sha512-VUjpoHg5/uNNkg1cTxWbUv+20t5CTAf1tB4dhBmGSYjgb+Efo5gi3aO/p3ajYuzOOMOxF+EhjQVouU6BD3E+Xg==", + "version": "3.37.0-beta.5", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.5.tgz", + "integrity": "sha512-dtHMzMU2Qa9c6DpygWr7fib19l1zqbYgBi2wGkiJ6uMAvl1CGu9V0p+JA1FwGTQ0QRy7EkkRVi6X6sH5u/ICpA==", + "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" @@ -3526,10 +3527,11 @@ "dev": true }, "node_modules/@types/multer": { - "version": "1.4.11", - "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.11.tgz", - "integrity": "sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w==", + "version": "1.4.12", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.12.tgz", + "integrity": "sha512-pQ2hoqvXiJt2FP9WQVLPRO+AmiIm/ZYkavPlIQnx282u4ZrVdztx0pkh3jjpQt0Kz+YI0YhSG264y08UJKoUQg==", "dev": true, + "license": "MIT", "dependencies": { "@types/express": "*" } diff --git a/package.json b/package.json index fbe7eea..072bfb6 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.34.0", + "@dadosfera/protospack-v2": "3.37.0-beta.5", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -74,7 +74,7 @@ "@types/jest": "27.0.2", "@types/jsonwebtoken": "^8.5.9", "@types/jwk-to-pem": "^2.0.1", - "@types/multer": "^1.4.7", + "@types/multer": "^1.4.12", "@types/node": "^16.18.52", "@types/passport-facebook": "^2.1.11", "@types/passport-google-oauth20": "^2.0.11", diff --git a/src/app.module.ts b/src/app.module.ts index fa831fd..3e54fde 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -26,9 +26,10 @@ import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module'; import { ProductboardModule } from './modules/productboard/productboard.module'; import { MixpanelModule } from './modules/mixpanel/mixpanel.module'; import { CustomersModule } from './modules/customers/customers.module'; +import { OpenDataModule } from './modules/open-data/open-data.module'; +import { ThemeModule } from './modules/theme/theme.module'; @Module({ - controllers: [], providers: [ DadosferaLogger, { @@ -58,8 +59,10 @@ import { CustomersModule } from './modules/customers/customers.module'; ProductboardModule, MixpanelModule, CustomersModule, + OpenDataModule, + ThemeModule, //Always leave HealthModule last, so it is on the bottom of swagger - HealthModule, + HealthModule ], }) export class AppModule {} diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index 33a779b..6e87606 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -77,6 +77,8 @@ export class AuthCustomer { scheduleLimit: string; @ApiProperty() links: Link[]; + @ApiProperty() + themeEnabled: boolean; } export class AuthSignInReq implements AuthSignInRequest { diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 04073ec..1bd9835 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -1,5 +1,4 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; -import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; import { Body, Controller, @@ -18,7 +17,6 @@ import { Authenticated, RequireAllPermissions, } from 'src/decorators/authentication.decorator'; -import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { CustomersService } from './customers.service'; import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers'; @@ -28,7 +26,6 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; @ApiTags('Customers') @Controller('customers') -@Authenticated() @UseFilters(GrpcToHttpExceptionFilter) export class CustomersController { logger: DadosferaLogger; @@ -42,6 +39,7 @@ export class CustomersController { } @Get(':id/links') + @Authenticated() @ApiOkResponse({ type: CustomerLinksResponse }) async getCustomerLinks(@Param('id') id: string) { this.logger.info('getCustomerLinks', { id }); @@ -50,6 +48,7 @@ export class CustomersController { } @Put(':id/links') + @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) @ApiOkResponse() @HttpCode(HttpStatus.OK) @@ -63,6 +62,7 @@ export class CustomersController { } @Get('token') + @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.AUTH.permissions.GENERATE_TOKEN) @ApiProduces('text/plain') async getCustomerToken( @@ -79,6 +79,7 @@ export class CustomersController { } @Get('monitoring-dashboard') + @Authenticated() @RequireAllPermissions( PERMISSIONS_GROUPS.CUSTOMER.permissions.MONITORING_DASHBOARD, ) diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 5102db5..cd7f472 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -28,19 +28,24 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { Metadata } from '@grpc/grpc-js'; import { PipelinesClientConfiguration } from '../pipelinesV2/pipelines-client'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; // This function will accept any string, which may result in a bug. @Injectable() export class CustomersService implements OnModuleInit { private customerService: CustomersProtoService; - + private logger: DadosferaLogger; private pipelineReadService: ReadService.PipelineV2ReadService; constructor( @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, @Inject(PipelinesClientConfiguration.name) private readonly pipelinesGrpcClient: ClientGrpc, - ) {} + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } onModuleInit() { this.customerService = this.grpcClient.getService( diff --git a/src/modules/customers/dtos/customers.ts b/src/modules/customers/dtos/customers.ts index c3b3070..db56ee3 100644 --- a/src/modules/customers/dtos/customers.ts +++ b/src/modules/customers/dtos/customers.ts @@ -20,3 +20,4 @@ export class CustomerLinksResponse { @ApiProperty({ type: [CustomerLink] }) links: CustomerLink[]; } + diff --git a/src/modules/duc/client.config.ts b/src/modules/duc/client.config.ts index 25ce28b..4cf4124 100644 --- a/src/modules/duc/client.config.ts +++ b/src/modules/duc/client.config.ts @@ -29,6 +29,8 @@ export class DucClient { objects: true, arrays: true, }, + maxSendMessageLength: 15 * 1024 * 1024, // 15 MB por mensagem + maxReceiveMessageLength: 15 * 1024 * 1024, }, }; diff --git a/src/modules/open-data/dto/wordpres-form.ts b/src/modules/open-data/dto/wordpres-form.ts new file mode 100644 index 0000000..39b6ed7 --- /dev/null +++ b/src/modules/open-data/dto/wordpres-form.ts @@ -0,0 +1,48 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class CreateUserOpenDataDTO { + @ApiProperty() + firstName: string; + @ApiProperty() + lastName: string; + @ApiProperty() + email: string; + @ApiProperty() + organization: string; + @ApiProperty() + enquiryType: string; +} + +type FormField = { + id: string; + type: string; + title: string; + value: string; + raw_value: string; + required: string; +}; + +type MetaData = { + title: string; + value: string; +}; + +export type WordpressForm = { + form: { + id: string; + name: string; + }; + fields: { + [key: string]: FormField; + }; + meta: { + date: MetaData; + time: MetaData; + page_url: MetaData; + user_agent: MetaData; + remote_ip: MetaData; + credit: MetaData; + }; +}; + + diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts new file mode 100644 index 0000000..b999060 --- /dev/null +++ b/src/modules/open-data/open-data.controller.ts @@ -0,0 +1,98 @@ +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { Body, Controller, ForbiddenException, Header, HttpCode, Inject, Param, Post, Query, Req, UseFilters, UseGuards, UseInterceptors } from '@nestjs/common'; +import { ApiCreatedResponse, ApiHeaders, ApiOkResponse, ApiTags } from '@nestjs/swagger'; +import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { LanguageEnum } from 'src/utils/languages.enum'; +import { UsersService } from '../users/users.service'; +import { Language } from 'src/decorators/language.decorator'; +import { OpenDataService } from './open-data.service'; +import { CreateUserOpenDataDTO, WordpressForm } from './dto/wordpres-form'; +import { Metadata } from '@grpc/grpc-js'; +import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { request } from 'http'; +import { Request } from 'express'; + +@Controller('open-data') +@ApiInternalOnlyController() +@ApiTags('OpenData') +@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) +@UseFilters(GrpcToHttpExceptionFilter) +export class OpenDataController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private openDataService: OpenDataService, + ) { + this.logger = dadosferaLogger.logger; + } + + @Post("/sharing-ocean-data") + @HttpCode(200) + @Header('content-type', 'application/json') + @ApiOkResponse() + async createUser( + @Body() + body: WordpressForm, + @Query('language') + language: string, + @Req() + request: Request, + ) { + if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) { + throw new ForbiddenException(); + } + this.logger.info('createUser for open data'); + + this.logger.info('user-agent', request.headers['user-agent']); + + const OPENDATA_CUSTOMER_ID = process.env.OPEN_CUSTOMER_ID; + const OPENDATA_GROUP_ID = process.env.OPEN_GROUP_ID; + const roles = [process.env.OPEN_GROUP_ID]; + const metadata = PackTheMetadata({ + language: language || 'en-us' + }); + + const data = {} + + try { + Object.keys(body.fields) + .filter(key => body.fields[key].required === "1") + .forEach(key => { + const field = body.fields[key] + data[field.id] = field.value + }); + } catch (e) { + this.logger.error('user data ' + e.message); + } + + const user: CreateUserOpenDataDTO = { + email: data["email"], + enquiryType: data["enquiry_type"], + firstName: data["first_name"], + lastName: data["last_name"], + organization: data["organization"] + } + this.logger.info(`user request to group ${OPENDATA_CUSTOMER_ID} with role ${OPENDATA_GROUP_ID}`); + + try { + const id = await this.openDataService.createUser(OPENDATA_CUSTOMER_ID, user, roles, metadata); + this.logger.info('user created with id: '+ id); + return { + success: true, + status: 'success', + message: 'user created with succesfull' + } + } catch (e) { + this.logger.error('failed with exception: ' + e.message); + return { + success: false, + status: 'failed', + message: e.message + }; + } + } + +} diff --git a/src/modules/open-data/open-data.module.ts b/src/modules/open-data/open-data.module.ts new file mode 100644 index 0000000..f82b125 --- /dev/null +++ b/src/modules/open-data/open-data.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; +import { OpenDataController } from './open-data.controller'; +import { UsersService } from '../users/users.service'; +import { ClientsModule } from '@nestjs/microservices' +import { DucClient } from '../duc/client.config'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { RolesModule } from '../roles/roles.module'; +import { PermissionsModule } from '../permissions/permissions.module'; +import { OpenDataService } from './open-data.service'; + +const client = new DucClient(); + +@Module({ + controllers: [OpenDataController], + imports: [ + ClientsModule.register([client.providerOptions]), + RolesModule, + // PermissionsModule, + ], + providers: [DadosferaLogger, UsersService, OpenDataService] +}) +export class OpenDataModule {} diff --git a/src/modules/open-data/open-data.service.ts b/src/modules/open-data/open-data.service.ts new file mode 100644 index 0000000..4c927fa --- /dev/null +++ b/src/modules/open-data/open-data.service.ts @@ -0,0 +1,51 @@ +import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { lastValueFrom } from 'rxjs'; +import { CreateUserOpenDataDTO } from './dto/wordpres-form'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { DucClient } from '../duc/client.config'; +import { ClientGrpc } from '@nestjs/microservices'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { Metadata } from '@grpc/grpc-js'; + +@Injectable() +export class OpenDataService implements OnModuleInit { + logger: DadosferaLogger; + + private usersClientService: UsersProtoService; + constructor( + @Inject(DadosferaLogger) + private dadosferaLogger: DadosferaLogger, + @Inject(DucClient.name) + private readonly grpcClient: ClientGrpc, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.usersClientService = this.grpcClient.getService( + ProtoServices.UsersProtoService, + ); + + } + + async createUser(customerId: string, data: CreateUserOpenDataDTO, roleIds: string[], metadata: Metadata) { + const body = { + email: data.email, + name: data.firstName + " " + data.lastName, + department: data.organization, + jobTitle: data.enquiryType, + customerId: customerId, + roleIds: roleIds + } + + try { + const { user } = await lastValueFrom( + this.usersClientService.SimpleUserCreate(body, metadata), + ); + return user.id; + } catch(err) { + return err; + } + } +} diff --git a/src/modules/theme/dtos/customers.ts b/src/modules/theme/dtos/customers.ts new file mode 100644 index 0000000..05f5305 --- /dev/null +++ b/src/modules/theme/dtos/customers.ts @@ -0,0 +1,47 @@ +import { Link, Theme } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + +export class CustomerLink implements Link { + @ApiProperty() + href: string; + @ApiProperty() + name: string; + @ApiProperty() + description: string; + @ApiPropertyOptional() + iconSrc: string; +} +export class CustomerLinkRequest { + @ApiProperty({ type: [CustomerLink] }) + links: CustomerLink[]; +} + +export class CustomerLinksResponse { + @ApiProperty({ type: [CustomerLink] }) + links: CustomerLink[]; +} + +export class CustomerTheme implements Theme { + @ApiProperty() + backgroundColor: string; + @ApiProperty() + textColor: string; + @ApiProperty() + logo: string; + @ApiProperty() + logoLogin: string; +} + +export class CustomerThemeResponse { + @ApiProperty() + theme: CustomerTheme; +} + +export class CustomerThemeRequest { + @ApiProperty() + displayName: string; + @ApiProperty() + backgroundColor: string; + @ApiProperty() + textColor: string; +} diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts new file mode 100644 index 0000000..327c5be --- /dev/null +++ b/src/modules/theme/theme.controller.ts @@ -0,0 +1,127 @@ +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { + Body, + Controller, + Get, + HttpException, + HttpStatus, + Inject, + Param, + Post, + Put, + Query, + UploadedFiles, + UseFilters, + UseInterceptors, + HttpCode +} from '@nestjs/common'; +import { ApiOkResponse, ApiTags } from '@nestjs/swagger'; +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { + Authenticated, + RequireAllPermissions, +} from 'src/decorators/authentication.decorator'; +import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; +import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; + +import ErrorCodes from 'src/utils/errorCodes'; +import { AnyFilesInterceptor } from '@nestjs/platform-express'; +import { ThemeService } from './theme.service'; + +@ApiTags('Theme') +@Controller('customers') +@UseFilters(GrpcToHttpExceptionFilter) +export class ThemeController { + logger: DadosferaLogger; + + constructor( + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + private themeService: ThemeService, + ) { + this.logger = dadosferaLogger.logger; + } + + + @Post('/:id/theme') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse({ type: CustomerThemeResponse }) + @UseInterceptors(AnyFilesInterceptor()) + @HttpCode(HttpStatus.OK) + async saveCustomertheme( + @Param('id') id: string, + @Body() data: CustomerThemeRequest, + @UploadedFiles() files: Array + ) { + + this.logger.info('saveCustomertheme' + JSON.stringify({ + id, + })); + + const logo = files.find(file => file.fieldname === 'logo'); + const logoLogin = files.find(file => file.fieldname === 'logoLogin'); + + this.validFileSize(logo); + this.validFileSize(logoLogin); + this.validMimeType(logo); + this.validMimeType(logoLogin); + + try { + const theme = await this.themeService.createThemeByCustomer(id, { + ...data, + logo, + logoLogin + }); + this.logger.info('saveCustomertheme' + JSON.stringify(theme)); + return theme; + } catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - saveCustomertheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - saveCustomertheme Unknown Error:' + err?.message); + return { theme: null }; + }; + } + } + + private validFileSize(file: Express.Multer.File) { + const maxFileSize = 10 * 1024 * 1024; // 10MB + + if (file && file.size > maxFileSize) { + throw new HttpException(`O Arquivo ${file.filename} possui mais de 10MB`, HttpStatus.BAD_REQUEST); + } + } + + private validMimeType(file: Express.Multer.File) { + const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png', 'image/svg+xml']; + + if (file && !mimeTypesValid.includes(file.mimetype)) { + throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, png ou svg`, HttpStatus.BAD_REQUEST); + } + } + + @Get('/:id/theme') + @ApiOkResponse({ type: CustomerThemeResponse }) + async getCustomerTheme(@Param('id') id: string) { + this.logger.info('getCustomerTheme with id' + id); + + try { + const data = await this.themeService.getThemeByCustomer(id); + this.logger.info('Success - getCustomerTheme'+ JSON.stringify(data)); + if (data?.theme) return data; + + return { theme: null }; + }catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) { + this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND'); + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + } else { + this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message); + return { theme: null }; + }; + } + } + +} diff --git a/src/modules/theme/theme.module.ts b/src/modules/theme/theme.module.ts new file mode 100644 index 0000000..4fc3d53 --- /dev/null +++ b/src/modules/theme/theme.module.ts @@ -0,0 +1,20 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import { ClientsModule } from '@nestjs/microservices'; +import { ThemeController } from './theme.controller'; +import { ThemeService } from './theme.service'; +import { DucClient } from '../duc/client.config'; + +const ducClient = new DucClient(); + +@Module({ + imports: [ + ClientsModule.register([ + ducClient.providerOptions, + ]), + ], + controllers: [ThemeController], + providers: [ThemeService, DadosferaLogger], + exports: [ThemeService], +}) +export class ThemeModule {} diff --git a/src/modules/theme/theme.service.ts b/src/modules/theme/theme.service.ts new file mode 100644 index 0000000..b62338b --- /dev/null +++ b/src/modules/theme/theme.service.ts @@ -0,0 +1,158 @@ +import { + OnModuleInit, + Inject, + Injectable, + HttpException, + HttpStatus, + InternalServerErrorException, +} from '@nestjs/common'; + +import { firstValueFrom, lastValueFrom, ReplaySubject } from 'rxjs'; +import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities'; +import { DucClient } from '../duc/client.config'; +import { ClientGrpc } from '@nestjs/microservices'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; +import { ThemeProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { ThemeRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages'; +import { CustomerThemeRequest, CustomerThemeResponse } from './dtos/customers'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { resolve } from 'path'; +import { Readable } from 'stream'; + +type Files = { + logo: Express.Multer.File, + logoLogin: Express.Multer.File, +} + +@Injectable() +export class ThemeService implements OnModuleInit { + private themeService: ThemeProtoService; + private logger: DadosferaLogger; + + constructor( + @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.themeService = this.grpcClient.getService( + ProtoServices.ThemeProtoService, + ); + } + + async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) { + if (!id) { + this.logger.error('Error - saveCustomertheme - not found id:' + id); + throw new HttpException(null, HttpStatus.BAD_REQUEST); + } + + const customerThemeRequest$ = new ReplaySubject(); + + customerThemeRequest$.next({ + customerId: id, + displayName: theme.displayName, + backgroundColor: theme.backgroundColor, + textColor: theme.textColor, + isMetadata: true, + filename: '', + chunk: Buffer.alloc(0) + }) + + if(theme.logo) { + await this.sendFile(theme.logo, customerThemeRequest$); + } + + if(theme.logoLogin) { + await this.sendFile(theme.logoLogin, customerThemeRequest$); + } + customerThemeRequest$.complete(); + + const stream = this.themeService.CustomerCreateTheme(customerThemeRequest$); + + return lastValueFrom(stream); + } + + async getThemeByCustomer(id: string): Promise { + if (!id) { + this.logger.error('Error - getCustomerTheme - not found id:' + id); + throw new HttpException(null, HttpStatus.BAD_REQUEST); + } + + const { theme } = await firstValueFrom( + this.themeService.CustomerGetTheme({ + id + }), + ); + + return { + theme + } + } + + private async sendFile(file: Express.Multer.File, stream$: ReplaySubject) { + const chunkSize = 4 * 1024 * 1024; + const bufferStream = new CustomBufferStream(file.buffer, chunkSize); + const parseMimitypeForExtension = { + 'image/jpeg': '.jpeg', + 'image/jpg': '.jpg', + 'image/png': '.png', + 'image/svg+xml': '.svg', + } + + const extension = parseMimitypeForExtension[file.mimetype]; + + return new Promise((resolve, reject) => { + bufferStream.on('data', (chunk) => { + const filename = file.fieldname.concat(extension); + stream$.next({ + customerId: '', + displayName: '', + backgroundColor: '', + textColor: '', + isMetadata: false, + filename: filename, + chunk: chunk + }); + }); + + bufferStream.on('end', () => { + resolve(file.filename) + }); + + bufferStream.on('error', (err) => { + reject(err); + }); + }); + } + +} + +class CustomBufferStream extends Readable { + buffer: Buffer; + offset: number; + chunkSize: number; + + constructor(buffer: Buffer, chunkSize: number) { + super({ highWaterMark: chunkSize }); // Configura o tamanho do chunk + this.buffer = buffer; + this.offset = 0; + this.chunkSize = chunkSize; + } + + _read() { + if (this.offset < this.buffer.length) { + const end = Math.min(this.offset + this.chunkSize, this.buffer.length); + + const copiedBuf = Uint8Array.prototype.slice.call(this.buffer); + const chunk = copiedBuf.slice(this.offset, end); + this.offset = end; + this.push(chunk); + } else { + this.push(null); + } + } +} From 24085058898cd22f5f1ec2d5b177a576e33aed86 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Mon, 3 Feb 2025 12:25:56 -0300 Subject: [PATCH 44/45] FIX: remove support for svg --- src/modules/theme/theme.controller.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modules/theme/theme.controller.ts b/src/modules/theme/theme.controller.ts index 327c5be..ed8e3a0 100644 --- a/src/modules/theme/theme.controller.ts +++ b/src/modules/theme/theme.controller.ts @@ -95,10 +95,10 @@ export class ThemeController { } private validMimeType(file: Express.Multer.File) { - const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png', 'image/svg+xml']; + const mimeTypesValid = ['image/jpeg', 'image/jpg', 'image/png']; if (file && !mimeTypesValid.includes(file.mimetype)) { - throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, png ou svg`, HttpStatus.BAD_REQUEST); + throw new HttpException(`O Arquivo ${file.fieldname} deve ser jpeg, jpg, ou png`, HttpStatus.BAD_REQUEST); } } From 87a8d741d7e826c9c2234c867687861db5609387 Mon Sep 17 00:00:00 2001 From: marcos-silva-rodrigues Date: Thu, 13 Feb 2025 13:55:37 -0300 Subject: [PATCH 45/45] UPDATE: remove user agent --- src/modules/open-data/open-data.controller.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/modules/open-data/open-data.controller.ts b/src/modules/open-data/open-data.controller.ts index b999060..65172e0 100644 --- a/src/modules/open-data/open-data.controller.ts +++ b/src/modules/open-data/open-data.controller.ts @@ -41,9 +41,6 @@ export class OpenDataController { @Req() request: Request, ) { - if (!request.headers['user-agent'].includes('WordPress/6.7.1; https://devsbm.dadosfera.io')) { - throw new ForbiddenException(); - } this.logger.info('createUser for open data'); this.logger.info('user-agent', request.headers['user-agent']);