[CI] add minikubekvm integration test in prow

Author: ComradeProgrammer

Makefile

@@ -1126,3 +1126,9 @@ get-dependency-version:
 .PHONY: _update-all
 _update-all:
 	@(cd hack && go run update/update_all/update_all.go)
+
+
+
+# targets for tests on prow
+include ./hack/prow/prow.mk
+

hack/prow/common.sh

@@ -0,0 +1,207 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The script expects the following env variables:
+# OS: The operating system
+# ARCH: The architecture
+# DRIVER: the driver to use for the test
+# CONTAINER_RUNTIME: the container runtime to use for the test
+# EXTRA_START_ARGS: additional flags to pass into minikube start
+# EXTRA_TEST_ARGS: additional flags to pass into go test
+# JOB_NAME: the name of the logfile and check name to update on github
+
+function print_test_info() {
+	echo ">> Starting at $(date)"
+	echo ""
+	echo "user:      $(whoami)"
+	echo "arch:      ${OS_ARCH}"
+	echo "build:     ${MINIKUBE_LOCATION}"
+	echo "driver:    ${DRIVER}"
+	echo "runtime:   ${CONTAINER_RUNTIME}"
+	echo "job:       ${JOB_NAME}"
+	echo "test home: ${TEST_HOME}"
+	echo "kernel:    $(uname -v)"
+	echo "uptime:    $(uptime)"
+	# Setting KUBECONFIG prevents the version check from erroring out due to permission issues
+	echo "kubectl:   $(env KUBECONFIG=${TEST_HOME} kubectl version --client)"
+	echo "docker:    $(docker version --format '{{ .Client.Version }}')"
+	echo "podman:    $(sudo podman version --format '{{.Version}}' || true)"
+	echo "go:        $(go version || true)"
+
+	case "${DRIVER}" in
+	kvm2)
+		echo "virsh:     $(virsh --version)"
+		;;
+	virtualbox)
+		echo "vbox:      $(vboxmanage --version)"
+		;;
+	vfkit)
+		echo "vfkit:     $(vfkit --version)"
+		;;
+	krunkit)
+		echo "krunkit:   $(krunkit --version)"
+		;;
+	esac
+
+	echo ""
+}
+
+function install_dependencies() {
+	# We need pstree for the restart cronjobs
+	if [ "$(uname)" != "Darwin" ]; then
+		sudo apt-get -y install lsof psmisc dnsutils
+	else
+		brew install pstree coreutils pidof
+		ln -s /usr/local/bin/gtimeout /usr/local/bin/timeout || true
+	fi
+	# install golang if not present
+	sudo hack/prow/installer/check_install_golang.sh /usr/local 1.24.5 || true
+	# install gotestsum if not present
+	GOROOT="/usr/local/go" hack/prow/installer/check_install_gotestsum.sh || true
+	# install gopogh
+	hack/prow/installer/check_install_gopogh.sh || true
+
+	# install jq
+	if ! type "jq" >/dev/null; then
+		echo ">> Installing jq"
+		if [ "${ARCH}" == "arm64" && "${OS}" == "linux" ]; then
+			sudo apt-get install jq -y
+		elif [ "${ARCH}" == "arm64" ]; then
+			echo "Unable to install 'jq' automatically for arm64 on Darwin, please install 'jq' manually."
+			exit 5
+		elif [ "${OS}" != "darwin" ]; then
+			curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && sudo install jq-linux64 /usr/local/bin/jq
+		else
+			curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 && sudo install jq-osx-amd64 /usr/local/bin/jq
+		fi
+	fi
+
+}
+
+function docker_setup() {
+
+	# clean all docker artifacts up
+	docker system prune -a --volumes -f || true
+	docker system df || true
+	docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
+
+	# read only token, never expires
+	#todo: do we need this token
+	# docker login -u minikubebot -p "$DOCKERHUB_READONLY_TOKEN"
+}
+
+function gvisor_image_build() {
+	# Build the gvisor image so that we can integration test changes to pkg/gvisor
+	chmod +x testdata/gvisor-addon
+	# skipping gvisor mac because ofg https://github.com/kubernetes/minikube/issues/5137
+	if [ "$(uname)" != "Darwin" ]; then
+		# Should match GVISOR_IMAGE_VERSION in Makefile
+		docker build -t gcr.io/k8s-minikube/gvisor-addon:2 -f testdata/gvisor-addon-Dockerfile ./testdata
+	fi
+}
+
+function run_gopogh() {
+    # todo: currently we do not save to gopogh db
+    echo "Not saving to DB"
+    gopogh -in "${JSON_OUT}" -out_html "${HTML_OUT}" -out_summary "${SUMMARY_OUT}" -name "${JOB_NAME}" -pr "${MINIKUBE_LOCATION}" -repo github.com/kubernetes/minikube/ -details "${COMMIT}:$(date +%Y-%m-%d)"
+
+}
+
+# this is where the script starts
+readonly OS_ARCH="${OS}-${ARCH}"
+readonly TEST_ROOT="${HOME}/minikube-integration"
+readonly TEST_HOME="${TEST_ROOT}/${MINIKUBE_LOCATION}-$$"
+
+export GOPATH="$HOME/go"
+export KUBECONFIG="${TEST_HOME}/kubeconfig"
+export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
+export MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
+
+readonly TIMEOUT=120m
+
+cp -r test/integration/testdata .
+cp out/gvisor-addon testdata/
+ls testdata
+
+# Add the out/ directory to the PATH, for using new drivers.
+export PATH="$(pwd)/out/":$PATH
+mkdir -p "${TEST_ROOT}"
+mkdir -p "${TEST_HOME}"
+export MINIKUBE_HOME="${TEST_HOME}/.minikube"
+export MINIKUBE_BIN="out/minikube-${OS_ARCH}"
+export E2E_BIN="out/e2e-${OS_ARCH}"
+
+install_dependencies
+docker_setup
+print_test_info
+gvisor_image_build
+
+readonly TEST_OUT="${TEST_HOME}/testout.txt"
+readonly JSON_OUT="${TEST_HOME}/test.json"
+readonly JUNIT_OUT="${TEST_HOME}/junit-unit.xml"
+readonly HTML_OUT="${TEST_HOME}/test.html"
+readonly SUMMARY_OUT="${TEST_HOME}/test_summary.json"
+
+touch "${TEST_OUT}"
+touch "${JSON_OUT}"
+touch "${JUNIT_OUT}"
+touch "${HTML_OUT}"
+touch "${SUMMARY_OUT}"
+
+e2e_start_time="$(date -u +%s)"
+echo ""
+echo ">> Starting ${E2E_BIN} at $(date)"
+set -x
+
+EXTRA_START_ARGS="${EXTRA_START_ARGS} --container-runtime=${CONTAINER_RUNTIME}"
+echo $PATH
+gotestsum --jsonfile "${JSON_OUT}" --junitfile="${JUNIT_OUT}" -f standard-verbose --raw-command -- \
+	go tool test2json -t \
+	${E2E_BIN} \
+	-minikube-start-args="--driver=${DRIVER} ${EXTRA_START_ARGS}" \
+	-test.timeout=${TIMEOUT} -test.v \
+	${EXTRA_TEST_ARGS} \
+	-binary="${MINIKUBE_BIN}" 2>&1 |
+	tee "${TEST_OUT}"
+
+result=${PIPESTATUS[0]} # capture the exit code of the first cmd in pipe.
+set +x
+echo ">> ${E2E_BIN} exited with ${result} at $(date)"
+echo ""
+
+# calculate the time took to finish running e2e binary test.
+e2e_end_time="$(date -u +%s)"
+elapsed=$(($e2e_end_time - $e2e_start_time))
+min=$(($elapsed / 60))
+sec=$(tail -c 3 <<<$((${elapsed}00 / 60)))
+elapsed=$min.$sec
+
+#todo: currently we skip gopogh upload , we shall add it back
+run_gopogh
+
+# according to prow's requirement, upload the test report to $ARTIFACTS
+cp ${TEST_OUT} .
+cp ${JSON_OUT} .
+cp ${JUNIT_OUT} .
+cp ${HTML_OUT} .
+cp ${SUMMARY_OUT} .
+if [[ $result -eq 0 ]]; then
+	echo "minikube: SUCCESS"
+else
+	echo "minikube: FAIL"
+fi
+
+exit "$result"

hack/prow/docker.json

@@ -0,0 +1,3 @@
+{
+    "Image": "debian"
+}

hack/prow/installer/check_install_docker.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux -o pipefail
+
+ARCH=${ARCH:=amd64}
+
+
+echo "Installing latest docker"
+curl -fsSL https://get.docker.com -o get-docker.sh
+sudo sh get-docker.sh
+rm get-docker.sh
+
+sudo usermod -aG docker minitest || true
+
+echo "Installing latest kubectl"
+curl -LO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl"
+sudo install ./kubectl /usr/local/bin/kubectl

hack/prow/installer/check_install_golang.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script requires two parameters:
+# $1. INSTALL_PATH: The path to install the golang binary
+# $2. GO_VERSION: The version of golang to install
+
+
+
+set -eux -o pipefail
+
+if (($# < 2)); then
+  echo "ERROR: given ! ($#) parameters but expected 2."
+  echo "USAGE: ./check_install_golang.sh INSTALL_PATH GO_VERSION" 
+  exit 1
+fi
+
+VERSION_TO_INSTALL=${2}
+INSTALL_PATH=${1}
+
+function current_arch() {
+  case $(arch) in
+  "x86_64" | "i386")
+     echo "amd64"
+  ;;
+  "aarch64" | "arm64")
+    echo "arm64"
+  ;;
+  *)
+    echo "unexpected arch: $(arch). use amd64" 1>&2
+    echo "amd64"
+  ;;
+  esac
+}
+
+ARCH=${ARCH:=$(current_arch)}
+
+# installs or updates golang if right version doesn't exists
+function check_and_install_golang() {
+  if ! go version &>/dev/null; then
+    echo "WARNING: No golang installation found in your environment."
+    install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
+    return
+  fi
+  
+  sudo chown -R $USER:$USER "$INSTALL_PATH"/go
+
+  # golang has been installed and check its version
+  if [[ $(go version | cut -d' ' -f 3) =~ go(([0-9]+)\.([0-9]+).([0-9]+)*) ]]; then
+    HOST_VERSION=${BASH_REMATCH[1]}
+    if [ $HOST_VERSION == $VERSION_TO_INSTALL ]; then
+      echo "go version on the host looks good : $HOST_VERSION"
+    else
+      echo "WARNING: expected go version to be $VERSION_TO_INSTALL but got $HOST_VERSION"
+      install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
+    fi
+  else
+    echo "ERROR: Failed to parse golang version: $(go version)"
+    return
+  fi
+}
+
+# install_golang takes two parameters version and path to install.
+function install_golang() {
+  local -r GO_VER="$1"
+  local -r GO_DIR="$2/go"
+  echo "Installing golang version: $GO_VER in $GO_DIR"
+
+  INSTALLOS=linux
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    INSTALLOS=darwin
+  fi
+
+  local -r GO_TGZ="go${GO_VER}.${INSTALLOS}-${ARCH}.tar.gz"
+  pushd /tmp
+
+  # using sudo because previously installed versions might have been installed by a different user.
+  sudo rm -rf "$GO_TGZ"
+  curl -qL -O "https://storage.googleapis.com/golang/$GO_TGZ"
+  sudo rm -rf "$GO_DIR"
+  sudo mkdir -p "$GO_DIR"
+  sudo tar -C "$GO_DIR" --strip-components=1 -xzf "$GO_TGZ"
+
+  popd >/dev/null
+  echo "installed in $GO_DIR: $($GO_DIR/bin/go version)"
+}
+
+check_and_install_golang

hack/prow/installer/check_install_gopogh.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this script requires golang to be installed
+
+set -eux -o pipefail
+
+go install github.com/medyagh/gopogh/cmd/gopogh@v0.29.0