diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f3341e4..e04dce3c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,15 +7,21 @@ All notable changes to this project will be documented in this file. ### Added - superset: Add 6.0.0-rc2 ([#1337]). +- hive: Build [hive-metastore-opa-authorizer](https://github.com/boschglobal/hive-metastore-opa-authorizer) from source and add to image ([#1340]). ### Changed - airflow: Extend list of providers for 3.0.6 ([#1336]) - airflow: Bump celery version to 5.5.3 for Airflow 3.x ([#1343]). +### Removed + +- hive: Remove `4.0.0` ([#1340]). + [#1336]: https://github.com/stackabletech/docker-images/pull/1336 [#1337]: https://github.com/stackabletech/docker-images/pull/1337 [#1343]: https://github.com/stackabletech/docker-images/pull/1343 +[#1340]: https://github.com/stackabletech/docker-images/pull/1340 ## [25.11.0] - 2025-11-07 diff --git a/hive/Dockerfile b/hive/Dockerfile index 46ab0412f..037bb0321 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -2,6 +2,7 @@ # check=error=true FROM local-image/hadoop/hadoop AS hadoop-builder +FROM local-image/hive/hive-metastore-opa-authorizer AS hive-metastore-opa-authorizer-builder FROM local-image/java-devel AS hive-builder @@ -44,6 +45,10 @@ ENV NEW_VERSION="${PRODUCT_VERSION}-stackable${RELEASE_VERSION}" # thus taking a bit (which is annoying while development) RUN /stackable/patchable --images-repo-root=src checkout hive ${PRODUCT_VERSION} > /tmp/HIVE_SOURCE_DIR +# Use bash for regex machting, otherwise docker lint is complaining: +# hive/Dockerfile:51 SC3014 warning: In POSIX sh, == in place of = is undefined. +SHELL ["/bin/bash", "-c"] + # Make expensive maven build a separate layer for better caching # Cache mounts are owned by root by default # We need to explicitly give the uid to use @@ -184,6 +189,8 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hive-${PRODU COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/*-src.tar.gz /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-metastore-opa-authorizer-builder /stackable/opa-authorizer-bin /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/lib +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-metastore-opa-authorizer-builder /stackable/opa-authorizer-src /stackable COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/jmx /stackable/jmx COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable/bin/start-metastore /stackable/apache-hive-metastore-${PRODUCT_VERSION}-stackable${RELEASE_VERSION}-bin/bin @@ -230,8 +237,8 @@ EOF USER ${STACKABLE_USER_UID} -ENV HADOOP_HOME=/stackable/hadoop ENV HIVE_HOME=/stackable/hive-metastore +ENV HADOOP_HOME=/stackable/hadoop ENV PATH="${PATH}":/stackable/hadoop/bin:/stackable/hive-metastore/bin # The following 2 env-vars are required for common hadoop scripts even if the respective libraries are never used. diff --git a/hive/boil-config.toml b/hive/boil-config.toml index 2c99809a8..f4d4ed316 100644 --- a/hive/boil-config.toml +++ b/hive/boil-config.toml @@ -3,6 +3,8 @@ java-base = "11" java-devel = "8" "hadoop/hadoop" = "3.3.6" +# hive-metastore-opa-authorizer from: https://github.com/boschglobal/hive-metastore-opa-authorizer +"hive/hive-metastore-opa-authorizer" = "v1.0.0-hive-3.1.3-hadoop-3.3.6" [versions."3.1.3".build-arguments] jmx-exporter-version = "1.3.0" @@ -11,24 +13,13 @@ aws-java-sdk-bundle-version = "1.12.367" azure-storage-version = "7.0.1" azure-keyvault-core-version = "1.0.0" -[versions."4.0.0".local-images] -# Hive 4 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 -java-base = "11" -java-devel = "8" -"hadoop/hadoop" = "3.3.6" - -[versions."4.0.0".build-arguments] -jmx-exporter-version = "1.3.0" -# Keep consistent with the dependency from Hadoop: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.6 -aws-java-sdk-bundle-version = "1.12.367" -azure-storage-version = "7.0.1" -azure-keyvault-core-version = "1.0.0" - [versions."4.0.1".local-images] # Hive 4.0 must be built with Java 8 (according to GitHub README) but seems to run on Java 11 java-base = "11" java-devel = "8" "hadoop/hadoop" = "3.3.6" +# hive-metastore-opa-authorizer from: https://github.com/boschglobal/hive-metastore-opa-authorizer +"hive/hive-metastore-opa-authorizer" = "v1.0.0-hive-4.0.1-hadoop-3.3.6" [versions."4.0.1".build-arguments] jmx-exporter-version = "1.3.0" @@ -42,6 +33,8 @@ azure-keyvault-core-version = "1.0.0" java-base = "17" java-devel = "17" "hadoop/hadoop" = "3.4.2" +# hive-metastore-opa-authorizer from: https://github.com/boschglobal/hive-metastore-opa-authorizer +"hive/hive-metastore-opa-authorizer" = "v1.0.0-hive-4.1.0-hadoop-3.4.2" [versions."4.1.0".build-arguments] jmx-exporter-version = "1.3.0" diff --git a/hive/hive-metastore-opa-authorizer/Dockerfile b/hive/hive-metastore-opa-authorizer/Dockerfile new file mode 100644 index 000000000..4c3f3bee2 --- /dev/null +++ b/hive/hive-metastore-opa-authorizer/Dockerfile @@ -0,0 +1,76 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# check=error=true + +FROM local-image/hadoop/hadoop AS hadoop-builder + +FROM local-image/java-devel + +ARG AUTHORIZER_VERSION +ARG HIVE_VERSION +ARG HADOOP_HADOOP_VERSION +# Reassign the arg to `HADOOP_VERSION` for better readability. +ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} +ARG STACKABLE_USER_UID +# Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) +# This can be used to speed up builds when disk space is of no concern. +ARG DELETE_CACHES="true" + +USER ${STACKABLE_USER_UID} +WORKDIR /stackable + +COPY --chown=${STACKABLE_USER_UID}:0 hive/hive-metastore-opa-authorizer/stackable/patches/patchable.toml /stackable/src/hive/hive-metastore-opa-authorizer/stackable/patches/patchable.toml +COPY --chown=${STACKABLE_USER_UID}:0 hive/hive-metastore-opa-authorizer/stackable/patches/${AUTHORIZER_VERSION} /stackable/src/hive/hive-metastore-opa-authorizer/stackable/patches/${AUTHORIZER_VERSION} + +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/patched-libs /stackable/patched-libs + +# Use bash for regex machting, otherwise docker lint is complaining: +# hive/hive-metastore-opa-authorizer/Dockerfile:26 SC3015 warning: In POSIX sh, =~ regex matching is undefined. +SHELL ["/bin/bash", "-c"] + +# Make expensive maven build a separate layer for better caching +# Cache mounts are owned by root by default +# We need to explicitly give the uid to use +RUN --mount=type=cache,id=maven-hive-metastore-opa-authorizer-${AUTHORIZER_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <