UPDATE: create helm chart for maestro

.github/workflows/deploy-k8s.yml

@@ -0,0 +1,73 @@
+name: Deploy K8S Modifications
+
+on:
+  push:
+    branches:
+      - main
+      - beta
+
+jobs:
+  extract_environment:
+    runs-on: ubuntu-22.04
+    outputs:
+      environment: ${{ steps.extract_environment.outputs.environment }}
+    steps:
+      - name: Extract Environment
+        run: |
+          if [ ${GITHUB_REF} == "refs/heads/main" ]; then
+            echo "environment=prd" >> $GITHUB_OUTPUT
+          elif [ ${GITHUB_REF} == "refs/heads/beta" ]; then
+            echo "environment=stg" >> $GITHUB_OUTPUT
+          fi
+        id: extract_environment
+
+  helmfile-deploy:
+    needs: [extract_environment]
+    runs-on: [self-hosted, "prd-azure"]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Install Azure ClI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+      - uses: azure/login@v2
+        with:
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: 'v1.30.1'
+
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
+
+      - name: Run Helmfile Apply
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: helmfile -f helmfiles/${ENV}.yaml sync

.github/workflows/validate-k8s.yml

@@ -0,0 +1,79 @@
+name: Validate K8S Modifications
+
+on:
+  pull_request:
+    branches:
+      - main
+      - stg
+
+jobs:
+  extract_environment:
+    runs-on: ubuntu-22.04
+    outputs:
+      environment: ${{ steps.extract_environment.outputs.environment }}
+    steps:
+      - name: Extract Environment
+        run: |
+          if [ "${{ github.event.pull_request.base.ref }}" == "main" ]; then
+            echo "environment=prd" >> $GITHUB_OUTPUT
+          elif [ "${{ github.event.pull_request.base.ref }}" == "beta" ]; then
+            echo "environment=stg" >> $GITHUB_OUTPUT
+          fi
+        id: extract_environment
+
+  helmfile-deploy:
+    needs: [extract_environment]
+    runs-on: [self-hosted, "prd-azure"]
+
+    steps:
+      - name: Summary
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: |
+          echo "### :rocket: Deploy da branch \`$GITHUB_REF_NAME\`  para o environment ($ENV)" >> $GITHUB_STEP_SUMMARY
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: 'v3.9.0'
+
+      - name: Install Azure ClI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+      - uses: azure/login@v2
+        with:
+          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install Helmfile
+        run: |
+          wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
+          tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
+          mv helmfile /usr/local/bin/
+          helmfile --version
+
+      - name: Install Helm Diff Plugin
+        run: helm plugin install https://github.com/databus23/helm-diff || true
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v1
+        with:
+          version: 'v1.30.1'
+
+      - name: Authenticate with cluster
+        env:
+          CLUSTER_NAME: platform-${{ needs.extract_environment.outputs.environment }}
+        run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
+
+      - name: Run Helmfile Diff
+        env:
+          ENV: ${{ needs.extract_environment.outputs.environment }}
+        run: helmfile -f helmfiles/${ENV}.yaml diff

helmfiles/prd.yaml

@@ -0,0 +1,16 @@
+charts:
+  - name: maestro
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: maestro.duc_url
+        value: duc.dadosfera.ai
+      - name: hostname
+        value: maestro.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory.dadosfera.ai

helmfiles/stg.yaml

@@ -0,0 +1,16 @@
+charts:
+  - name: maestro
+    chart: ../maestro
+    values:
+    - ../maestro/values.yaml
+    set:
+      - name: maestro.duc_url
+        value: duc-temp.dadosfera.ai
+      - name: hostname
+        value: maestro-temp.dadosfera.ai
+      - name: maestro.pi_factory_url
+        value: pi-factory-temp.dadosfera.ai
+      - name: maestro.in_factory_url
+        value: in-factory-temp.dadosfera.ai
+      - name: maestro.tr_factory_url
+        value: in-factory-temp.dadosfera.ai

maestro/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

maestro/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: maestro
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

maestro/templates/deployment.yaml

@@ -0,0 +1,112 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: maestro
+  namespace: applications
+  labels:
+    app: maestro
+
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: maestro
+  
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 25%
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: maestro
+    
+    spec:
+      imagePullSecrets:
+        - name: {{ .Values.imagePullSecrets }}
+      nodeSelector:
+        "beta.kubernetes.io/os": linux
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: application
+                    operator: In
+                    values:
+                      - backend
+
+      containers:
+        - name: maestro
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+          ports:
+            - containerPort: {{ .Values.containerPort }}
+          resources:
+            requests:
+              cpu: {{ .Values.resources.requests.cpu }}
+              memory: {{ .Values.resources.requests.memory }}
+            limits:
+              cpu: {{ .Values.resources.limits.cpu }}
+              memory: {{ .Values.resources.limits.memory }}
+          env:
+            - name: AWS_IDENTITY_POOL_ID
+              value: {{ .Values.maestro.aws_identity_pool_id }}
+            - name: AWS_REGION
+              value: "us-east-1"
+            - name: BASE_HOST
+              value: "maestro_prd"
+            - name: BUCKET_CUSTOMER_CSV_ASSETS
+              value: {{ .Values.maestro.bucket_customer_csv_assets }}
+            - name: CONNECTORS_INDEX
+              value: "connectors"
+            - name: DUC_URL
+              value: {{ .Values.maestro.duc_url }}
+            - name: ELASTIC_APM_ENVIRONMENT
+              value: {{ .Values.maestro.env }}
+            - name: ELASTIC_APM_SERVER_URL
+              value: "https://apm-server.dadosfera.ai"
+            - name: ELASTIC_APM_SERVICE_NAME
+              value: {{ .Values.maestro.apm_service }}
+            - name: ENV
+              value: {{ .Values.maestro.env }}
+            - name: INFACTORY_URL
+              value: {{ .Values.maestro.in_factory_url }}
+            - name: LOGGER_GELF_HOST
+              value: "logstash-pipelines.dadosfera.ai"
+            - name: LOGGER_GELF_PORT
+              value: "{{ .Values.maestro.logger_gelf_port }}"
+            - name: NIMBUS_BASE_URL
+              value: "http://nimbus-api"
+            - name: NPM_TOKEN
+              value: {{ .Values.maestro.npm_token }}
+            - name: PB_TOKEN_PATH
+              value: {{ .Values.maestro.pb_token_path }}
+            - name: PIFACTORY_URL
+              value: {{ .Values.maestro.pi_factory_url }}
+            - name: SM_OAUTH_PATH
+              value: {{ .Values.maestro.sm_oauth_path }}
+            - name: TRFACTORY_URL
+              value: {{ .Values.maestro.tr_factory_url }}
+            - name: UPLOAD_FILE_AGENT_CONNECTION
+              value: {{ .Values.maestro.upload_file_agent_connection }}
+            - name: JWT_PRIVATE_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: prd-duc
+                  key: jwt_token
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: prd-maestro
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: prd-maestro
+                  key: AWS_SECRET_ACCESS_KEY
+            - name: AWS_DEFAULT_REGION
+              valueFrom:
+                secretKeyRef:
+                  name: prd-maestro
+                  key: AWS_DEFAULT_REGION

maestro/templates/ingress.yaml

@@ -0,0 +1,27 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/server-snippet: |
+      underscores_in_headers on;
+      ignore_invalid_headers on;
+      
+  generation: 1
+  labels:
+    app: maestro
+  name: maestro
+  namespace: applications
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: {{ .Values.hostname }}
+    http:
+      paths:
+      - backend:
+          service:
+            name: maestro
+            port:
+              number: {{ .Values.ingress.port }}
+        path: /
+        pathType: Prefix

maestro/templates/secret.yaml

@@ -0,0 +1,40 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: prd-maestro
+  namespace: applications
+  labels:
+    app: maestro
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: secretsmanager-prd
+    kind: SecretStore
+  target:
+    name: prd-maestro
+    creationPolicy: Owner
+  data:
+  - secretKey: AWS_ACCESS_KEY_ID
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_ACCESS_KEY_ID
+
+  - secretKey: AWS_SECRET_ACCESS_KEY
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_SECRET_ACCESS_KEY
+
+  - secretKey: AWS_DEFAULT_REGION
+    remoteRef:
+      key: prd/microservices/aws_credentials/maestro
+      version: "AWSCURRENT"
+      property: AWS_DEFAULT_REGION
+
+  - secretKey: jwt_token
+    remoteRef:
+      key: {{ .Values.maestro.jwt_token_secret_id }}
+      version: "AWSCURRENT"
+      property: token
+

maestro/templates/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: maestro
+  namespace: applications
+  labels:
+    app: maestro
+
+spec:
+  type: ClusterIP
+  ports:
+  - name: maestro
+    protocol: TCP
+    port: {{ .Values.service.port }}
+    targetPort: {{ .Values.service.targetPort }}
+  selector:
+    app: maestro
+

maestro/values.yaml

@@ -0,0 +1,49 @@
+# Default values for metabase.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 3
+hostname: maestro-temp.dadosfera.ai
+image:
+  repository: 611330257153.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_prd
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: 1.56.0
+containerPort: 3333
+imagePullSecrets: "applications-secrets-ecr-auth-token-external-secret"
+service:
+  type: ClusterIP
+  port: 3333
+  targetPort: 3333
+ingress:
+  enabled: false
+  port: 3333
+resources:
+  requests:
+    cpu: 100m
+    memory: 1500Mi
+  limits:
+    cpu: 2000m
+    memory: 2Gi
+maestro:
+  aws_identity_pool_id: "us-east-1_Mrezsw9Sn"
+  duc_url: duc.dadosfera.ai
+  in_factory_url: in-factory.dadosfera.ai
+  bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153"
+  env: prd
+  apm_service: maestro
+  jwt_token_secret_id: prd/root/jwt_token
+  logger_gelf_port: "1026"
+  npm_token: npm_Xp17h3daMkORcZ55NY95Ez4gRfdzsQ3UvINP
+  pb_token_path: prd/root/productboard_token
+  pi_factory_url: pi-factory.dadosfera.ai
+  sm_oauth_path: prd/root/oauth_applications
+  tr_factory_url: in-factory.dadosfera.ai
+  upload_file_agent_connection: cbc2f881-58c4-4d60-8003-0979b0b5b911
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  targetCPUUtilizationPercentage: 80
+  targetMemoryUtilizationPercentage: 80