refactor: use OSS Pulsar image and charts for e2e tests

.ci/clusters/values.yaml

@@ -22,54 +22,43 @@ affinity:
 
 # disable auto recovery
 components:
+  functions: true
   autorecovery: false
   pulsar_manager: false
-  sql_worker: false
   proxy: false
   toolset: false
 
 ## disable monitoring stack
-monitoring:
-  # monitoring - prometheus
-  prometheus: false
-  # monitoring - grafana
-  grafana: false
-  # monitoring - node_exporter
-  node_exporter: false
-  # alerting - alert-manager
-  alert_manager: false
-  # monitoring - loki
-  loki: false
-  # monitoring - datadog
-  datadog: false
+victoria-metrics-k8s-stack:
+  enabled: false
 
 images:
   zookeeper:
-    repository: streamnative/sn-platform
-    tag: 2.10.4.3
+    repository: apachepulsar/pulsar-all
+    tag: 4.1.2
   bookie:
-    repository: streamnative/sn-platform
-    tag: 2.10.4.3
+    repository: apachepulsar/pulsar-all
+    tag: 4.1.2
   broker:
-    repository: streamnative/sn-platform
-    tag: 2.10.4.3
+    repository: apachepulsar/pulsar-all
+    tag: 4.1.2
   functions:
-    repository: streamnative/sn-platform
-    tag: 2.10.4.3
+    repository: apachepulsar/pulsar-all
+    tag: 4.1.2
 
 zookeeper:
   replicaCount: 1
+  podMonitor:
+    enabled: false
   resources:
     requests:
       memory: 256Mi
       cpu: 10m
 
 bookkeeper:
   replicaCount: 1
-  metadata:
-    image:
-      repository: streamnative/sn-platform
-      tag: 2.10.4.3
+  podMonitor:
+    enabled: false
   resources:
     requests:
       memory: 256Mi
@@ -94,7 +83,6 @@ bookkeeper:
       -XX:MaxGCPauseMillis=10
       -XX:+ParallelRefProcEnabled
       -XX:+UnlockExperimentalVMOptions
-      -XX:+AggressiveOpts
       -XX:+DoEscapeAnalysis
       -XX:ParallelGCThreads=4
       -XX:ConcGCThreads=4
@@ -106,11 +94,13 @@ bookkeeper:
 
 pulsar_metadata:
   image:
-    repository: streamnative/sn-platform
-    tag: 2.10.4.3
+    repository: apachepulsar/pulsar-all
+    tag: 4.1.2
 
 broker:
   replicaCount: 1
+  podMonitor:
+    enabled: false
   configData:
     ## Enable `autoSkipNonRecoverableData` since bookkeeper is running
     ## without persistence
@@ -119,16 +109,22 @@ broker:
     managedLedgerDefaultEnsembleSize: "1"
     managedLedgerDefaultWriteQuorum: "1"
     managedLedgerDefaultAckQuorum: "1"
-    enablePackagesManagement: "true"
     PULSAR_PREFIX_enablePackagesManagement: "true"
+    PULSAR_PREFIX_packagesManagementStorageProvider: org.apache.pulsar.packages.management.storage.bookkeeper.BookKeeperPackagesStorageProvider
+    PULSAR_PREFIX_packagesReplicas: "1"
+    PULSAR_PREFIX_packagesManagementLedgerRootPath: /ledgers
+    PULSAR_PREFIX_functionsWorkerEnablePackageManagement: "true"
     PULSAR_PREFIX_topicLevelPoliciesEnabled: "true"
   resources:
     requests:
       memory: 256Mi
       cpu: 10m
 
-functions:
-  functionState: false
-  useDedicatedRunner: false
-  configData:
-    narExtractionDirectory: /pulsar/data
+autorecovery:
+  podMonitor:
+    enabled: false
+
+proxy:
+  podMonitor:
+    enabled: false
+

.github/workflows/e2e_test.yml

@@ -34,8 +34,9 @@ jobs:
       ALWAYS_UPDATE_PULSAR_RESOURCE: ${{ matrix.alwaysUpdatePulsar }}
       GOPRIVATE: github.com/streamnative
       ACCESS_TOKEN: ${{ secrets.SNBOT_GITHUB_TOKEN }}
-      IMAGE: streamnative/sn-platform:2.10.4.3
+      IMAGE: apachepulsar/pulsar-all:4.1.2
       WATCH_CERT_MANAGER_CRDS: "false"
+      K8S_VERSION: v1.34.2
     steps:
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@v1.3.0
@@ -72,14 +73,13 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      # TODO the k8s version should be configurable
       - name: Setup K8s cluster
         run: |
-          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64
+          curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64
           chmod +x ./kind
           export PATH="$PWD:$PATH"
           kind version
-          ./hack/kind-cluster-build.sh --nodeNum 1 --k8sVersion v1.22.17
+          ./hack/kind-cluster-build.sh --nodeNum 1 --k8sVersion "$K8S_VERSION"
 
       - name: Initialize K8s cluster
         run: |
@@ -102,17 +102,14 @@ jobs:
         run: |
           helm repo add streamnative https://charts.streamnative.io
           helm repo add jetstack https://charts.jetstack.io
+          helm repo add apache https://pulsar.apache.org/charts
           helm repo update
-          helm install cert-manager jetstack/cert-manager --set installCRDs=true --version v1.8.2
-
+          helm install cert-manager jetstack/cert-manager --set crds.enabled=true --version v1.19.2
           rm -rf pulsar-charts/
-          git clone --branch pulsar-operator-0.17.10 https://github.com/streamnative/charts.git pulsar-charts
+          git clone --branch pulsar-4.4.0 https://github.com/apache/pulsar-helm-chart pulsar-charts
           cd pulsar-charts/
           ./scripts/pulsar/prepare_helm_release.sh -n default -k test -c
-          helm repo add grafana https://grafana.github.io/helm-charts
-          helm repo update
-          helm dependency update charts/pulsar
-          helm install test --set initialize=true --values ../.ci/clusters/values.yaml charts/pulsar
+          helm install --set initialize=true --values ../.ci/clusters/values.yaml test apache/pulsar
           df -h
 
       - name: Run Operator Test

.github/workflows/golangci-lint.yml

@@ -45,10 +45,10 @@ jobs:
 
       - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
+        uses: golangci/golangci-lint-action@v8
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.64
+          version: v2.7.2
           args: --timeout=5m
 
           # Optional: working directory, useful for monorepos

.gitignore

@@ -52,3 +52,7 @@ node_modules/
 
 .cursor
 .envrc
+
+docker-compose.yaml
+mise.toml
+.jj/