From b3c9cd2a37b3c9beef3f5314ebb2164ae5b37bfe Mon Sep 17 00:00:00 2001 From: donoghuc Date: Mon, 27 Oct 2025 15:31:24 -0700 Subject: [PATCH 1/4] Use logstash-versions.yml for observabilitySRE stack versions Previously the version of logstash checked in to the logstash repo was used to determine stack version for smoke and acceptance testing of the observabilitySRE artifact. Given the ES/Filebeat containers are subject to a unified release the delay between a logstash version bump and those artifacts being available was causing failures when the smoke/acceptance tests were run. In plugin testing the way we deal with this is to ensure the logstash-versions.yml file is updated when stack components are ready. This commit introduces the use of that pattern to the observabilitySRE artifact testing. The important step will be to set the STACK_VERSION to correspond to the desired stack version. For example the 9.2 logstash branch should look up `9.current`. --- ci/observabilitySREacceptance_tests.sh | 11 +++++++---- ci/observabilitySREsmoke_tests.sh | 8 ++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ci/observabilitySREacceptance_tests.sh b/ci/observabilitySREacceptance_tests.sh index b999ce1a064..6d8b7ac8efd 100755 --- a/ci/observabilitySREacceptance_tests.sh +++ b/ci/observabilitySREacceptance_tests.sh @@ -2,9 +2,12 @@ set -e -QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)" -export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$QUALIFIED_VERSION}" -export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$QUALIFIED_VERSION}" -export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$QUALIFIED_VERSION}" +# Look up corresponding LOSTASH_STREAM to test against. MAKE SURE TO UDPATE this per branch for example the 9.2 branch in logstash would +# correspond to the '9.current' key in the logstash-versions.yml file +LOGSTASH_STREAM=main +STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_STREAM}']") +export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$STACK_VERSION}" +export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}" +export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}" ./gradlew observabilitySREacceptanceTests --stacktrace diff --git a/ci/observabilitySREsmoke_tests.sh b/ci/observabilitySREsmoke_tests.sh index 744b2a79efa..4cf0f2dbbed 100755 --- a/ci/observabilitySREsmoke_tests.sh +++ b/ci/observabilitySREsmoke_tests.sh @@ -3,8 +3,12 @@ set -e QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)" -export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$QUALIFIED_VERSION}" -export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$QUALIFIED_VERSION}" +# Look up corresponding LOSTASH_STREAM to test against. MAKE SURE TO UDPATE this per branch for example the 9.2 branch in logstash would +# correspond to the '9.current' key in the logstash-versions.yml file +LOGSTASH_STREAM=main +STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_STREAM}']") +export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}" +export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}" ./gradlew --stacktrace artifactDockerObservabilitySRE -PfedrampHighMode=true From fda69b952388a081cc2d4f09e1a3edddf93795cd Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 29 Oct 2025 16:22:22 -0700 Subject: [PATCH 2/4] track stream in versions.yml --- ci/observabilitySREacceptance_tests.sh | 6 +++--- ci/observabilitySREsmoke_tests.sh | 6 +++--- versions.yml | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ci/observabilitySREacceptance_tests.sh b/ci/observabilitySREacceptance_tests.sh index 6d8b7ac8efd..ec20afe36fc 100755 --- a/ci/observabilitySREacceptance_tests.sh +++ b/ci/observabilitySREacceptance_tests.sh @@ -2,9 +2,9 @@ set -e -# Look up corresponding LOSTASH_STREAM to test against. MAKE SURE TO UDPATE this per branch for example the 9.2 branch in logstash would -# correspond to the '9.current' key in the logstash-versions.yml file -LOGSTASH_STREAM=main +# Look up corresponding LOGSTASH_STREAM from versions.yml +LOGSTASH_STREAM=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-stream']") +# Use logstash stream to find the corresponding stack verstion from logstash-versions.yml STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_STREAM}']") export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$STACK_VERSION}" export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}" diff --git a/ci/observabilitySREsmoke_tests.sh b/ci/observabilitySREsmoke_tests.sh index 4cf0f2dbbed..5a173b46a41 100755 --- a/ci/observabilitySREsmoke_tests.sh +++ b/ci/observabilitySREsmoke_tests.sh @@ -3,9 +3,9 @@ set -e QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)" -# Look up corresponding LOSTASH_STREAM to test against. MAKE SURE TO UDPATE this per branch for example the 9.2 branch in logstash would -# correspond to the '9.current' key in the logstash-versions.yml file -LOGSTASH_STREAM=main +# Look up corresponding LOGSTASH_STREAM from versions.yml +LOGSTASH_STREAM=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-stream']") +# Use logstash stream to find the corresponding stack verstion from logstash-versions.yml STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_STREAM}']") export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}" export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}" diff --git a/versions.yml b/versions.yml index 02422e9021a..a2a4bf2caeb 100644 --- a/versions.yml +++ b/versions.yml @@ -3,6 +3,8 @@ logstash: 9.3.0 logstash-core: 9.3.0 logstash-core-plugin-api: 2.1.16 +# Logstash stream corresponding to keys in https://github.com/logstash-plugins/.ci/blob/1.x/logstash-versions.yml +logstash-stream: main bundled_jdk: # for AdoptOpenJDK/OpenJDK jdk-14.0.1+7.1, the revision is 14.0.1 while the build is 7.1 From 300124149bce8659b9aca5163135b2e6fa367e14 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Fri, 31 Oct 2025 12:14:45 -0700 Subject: [PATCH 3/4] Finalize name/location for versions.yml mapping --- .ci/updatecli/bump-logstash-version.yml | 10 ++++++++++ .github/workflows/bump-logstash.yml | 6 +++++- ci/observabilitySREacceptance_tests.sh | 6 +++--- ci/observabilitySREsmoke_tests.sh | 6 +++--- versions.yml | 2 +- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.ci/updatecli/bump-logstash-version.yml b/.ci/updatecli/bump-logstash-version.yml index a100a19aeb8..1dc4186eca3 100644 --- a/.ci/updatecli/bump-logstash-version.yml +++ b/.ci/updatecli/bump-logstash-version.yml @@ -56,6 +56,16 @@ targets: key: $.logstash-core value: '{{ requiredEnv "LOGSTASH_VERSION" }}' + update_release_track: + name: Update logstash-release-track in versions.yml + kind: yaml + disablesourceinput: true + scmid: default + spec: + file: versions.yml + key: $.logstash-release-track + value: '{{ requiredEnv "LOGSTASH_RELEASE_TRACK" }}' + update_gemfile_lock_dependency: name: Update logstash-core dependency in lockfile kind: file diff --git a/.github/workflows/bump-logstash.yml b/.github/workflows/bump-logstash.yml index 6679f1ed618..13756814cac 100644 --- a/.github/workflows/bump-logstash.yml +++ b/.github/workflows/bump-logstash.yml @@ -11,7 +11,10 @@ on: description: 'Logstash branch (example: 9.1)' required: true type: string - + logstash_release_track: + description: 'Release track (example: 9.current, 9.previous, main)' + required: true + type: string jobs: bump: permissions: @@ -29,3 +32,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} LOGSTASH_BRANCH: "${{ github.event.inputs.logstash_branch }}" LOGSTASH_VERSION: "${{ github.event.inputs.logstash_version }}" + LOGSTASH_RELEASE_TRACK: "${{ github.event.inputs.logstash_release_track }}" diff --git a/ci/observabilitySREacceptance_tests.sh b/ci/observabilitySREacceptance_tests.sh index ec20afe36fc..24eb05769f9 100755 --- a/ci/observabilitySREacceptance_tests.sh +++ b/ci/observabilitySREacceptance_tests.sh @@ -2,10 +2,10 @@ set -e -# Look up corresponding LOGSTASH_STREAM from versions.yml -LOGSTASH_STREAM=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-stream']") +# Look up corresponding LOGSTASH_RELEASE_TRACK from versions.yml +LOGSTASH_RELEASE_TRACK=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-release-track']") # Use logstash stream to find the corresponding stack verstion from logstash-versions.yml -STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_STREAM}']") +STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_RELEASE_TRACK}']") export OBSERVABILITY_SRE_IMAGE_VERSION="${OBSERVABILITY_SRE_IMAGE_VERSION:-$STACK_VERSION}" export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}" export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}" diff --git a/ci/observabilitySREsmoke_tests.sh b/ci/observabilitySREsmoke_tests.sh index 5a173b46a41..bdd475b5dd1 100755 --- a/ci/observabilitySREsmoke_tests.sh +++ b/ci/observabilitySREsmoke_tests.sh @@ -3,10 +3,10 @@ set -e QUALIFIED_VERSION="$(.buildkite/scripts/common/qualified-version.sh)" -# Look up corresponding LOGSTASH_STREAM from versions.yml -LOGSTASH_STREAM=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-stream']") +# Look up corresponding LOGSTASH_RELEASE_TRACK from versions.yml +LOGSTASH_RELEASE_TRACK=$(ruby -ryaml -e "puts YAML.load_file('versions.yml')['logstash-release-track']") # Use logstash stream to find the corresponding stack verstion from logstash-versions.yml -STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_STREAM}']") +STACK_VERSION=$(ruby -ryaml -ropen-uri -e "puts YAML.load(URI.open('https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml'))['snapshots']['${LOGSTASH_RELEASE_TRACK}']") export ELASTICSEARCH_IMAGE_VERSION="${ELASTICSEARCH_IMAGE_VERSION:-$STACK_VERSION}" export FILEBEAT_IMAGE_VERSION="${FILEBEAT_IMAGE_VERSION:-$STACK_VERSION}" diff --git a/versions.yml b/versions.yml index a2a4bf2caeb..aebfbf170ef 100644 --- a/versions.yml +++ b/versions.yml @@ -4,7 +4,7 @@ logstash: 9.3.0 logstash-core: 9.3.0 logstash-core-plugin-api: 2.1.16 # Logstash stream corresponding to keys in https://github.com/logstash-plugins/.ci/blob/1.x/logstash-versions.yml -logstash-stream: main +logstash-release-track: main bundled_jdk: # for AdoptOpenJDK/OpenJDK jdk-14.0.1+7.1, the revision is 14.0.1 while the build is 7.1 From 9ec398cc17530ec514646a0f80fa8277ebb6ecb1 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Tue, 4 Nov 2025 10:29:40 -0800 Subject: [PATCH 4/4] better comment --- versions.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/versions.yml b/versions.yml index aebfbf170ef..c60a0b57ec8 100644 --- a/versions.yml +++ b/versions.yml @@ -3,7 +3,8 @@ logstash: 9.3.0 logstash-core: 9.3.0 logstash-core-plugin-api: 2.1.16 -# Logstash stream corresponding to keys in https://github.com/logstash-plugins/.ci/blob/1.x/logstash-versions.yml +# Logstash release track corresponding to keys in https://github.com/logstash-plugins/.ci/blob/1.x/logstash-versions.yml +# For example 9.current, 9.previous, etc logstash-release-track: main bundled_jdk: