Add gdeflate compression and decompression with NVIDIA fork of libdeflate.

BUILD.bazel

@@ -70,6 +70,9 @@ expand_template(
     out = "src/lib/OpenEXR/OpenEXRConfigInternal.h",
     substitutions = {
         "#cmakedefine OPENEXR_USE_INTERNAL_DEFLATE 1": "#define OPENEXR_USE_INTERNAL_DEFLATE 0",
+        "#cmakedefine OPENEXR_ENABLE_GDEFLATE 1": "/* #undef OPENEXR_ENABLE_GDEFLATE */",
+        # For gdeflate, set `_enable_gdeflate` to True in `MODULE.bazel` and replace the above line with:
+        # "#cmakedefine OPENEXR_ENABLE_GDEFLATE 1": "#define OPENEXR_ENABLE_GDEFLATE 1",
         "#cmakedefine OPENEXR_IMF_HAVE_COMPLETE_IOMANIP 1": "#define OPENEXR_IMF_HAVE_COMPLETE_IOMANIP 1",
         "#cmakedefine OPENEXR_IMF_HAVE_DARWIN 1": "/* #undef OPENEXR_IMF_HAVE_DARWIN */",
         "#cmakedefine OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX 1": "/* #undef OPENEXR_IMF_HAVE_GCC_INLINE_ASM_AVX */",
@@ -193,6 +196,7 @@ cc_library(
         "src/lib/OpenEXRCore/internal_dwa_simd.h",
         "src/lib/OpenEXRCore/internal_file.h",
         "src/lib/OpenEXRCore/internal_float_vector.h",
+        "src/lib/OpenEXRCore/internal_gdeflate_wrapper.h",
         "src/lib/OpenEXRCore/internal_ht.cpp",
         "src/lib/OpenEXRCore/internal_ht_common.h",
         "src/lib/OpenEXRCore/internal_ht_common.cpp",
@@ -291,6 +295,7 @@ cc_library(
         "src/lib/OpenEXR/ImfCompression.cpp",
         "src/lib/OpenEXR/ImfCompressionAttribute.cpp",
         "src/lib/OpenEXR/ImfCompressor.cpp",
+        "src/lib/OpenEXR/ImfGdeflateCompressor.cpp",
         "src/lib/OpenEXR/ImfContext.cpp",
         "src/lib/OpenEXR/ImfContextInit.cpp",
         "src/lib/OpenEXR/ImfConvert.cpp",
@@ -395,6 +400,7 @@ cc_library(
         "src/lib/OpenEXR/ImfCompression.h",
         "src/lib/OpenEXR/ImfCompressionAttribute.h",
         "src/lib/OpenEXR/ImfCompressor.h",
+        "src/lib/OpenEXR/ImfGdeflateCompressor.h",
         "src/lib/OpenEXR/ImfContext.h",
         "src/lib/OpenEXR/ImfContextInit.h",
         "src/lib/OpenEXR/ImfConvert.h",

MODULE.bazel

@@ -6,6 +6,19 @@ module(
     compatibility_level = 1,
 )
 
+# Toggle this to fetch the NVIDIA libdeflate fork, then update OPENEXR_ENABLE_GDEFLATE in BUILD.bazel.
+_enable_gdeflate = False
+
+_enable_gdeflate and archive_override(
+    module_name = "libdeflate",
+    patches = [
+        "//bazel:libdeflate_gdeflate_add_build_file.patch",
+        "//bazel:libdeflate_module_dot_bazel.patch",
+    ],
+    strip_prefix = "libdeflate-gdeflate",
+    urls = ["https://github.com/NVIDIA/libdeflate/archive/refs/heads/gdeflate.zip"],
+)
+
 bazel_dep(name = "bazel_skylib", version = "1.8.2")
 bazel_dep(name = "imath", version = "3.2.2")
 bazel_dep(name = "libdeflate", version = "1.24")

cmake/OpenEXRConfigInternal.h.in

@@ -17,6 +17,10 @@
 // deflate or the system provided version
 #cmakedefine OPENEXR_USE_INTERNAL_DEFLATE 1
 
+//
+// Whether libdeflate has gdeflate support (NVIDIA fork)
+#cmakedefine OPENEXR_ENABLE_GDEFLATE 1
+
 //
 // Define and set to 1 if the target system supports a proc filesystem
 // compatible with the Linux kernel's proc filesystem.  Note that this

cmake/OpenEXRSetup.cmake

@@ -202,9 +202,14 @@ endif()
 set (ILMTHREAD_USE_TBB ${OPENEXR_USE_TBB})
 
 option(OPENEXR_FORCE_INTERNAL_DEFLATE "Force using an internal libdeflate" OFF)
+option(OPENEXR_FORCE_FETCHCONTENT_DEFLATE "Force fetching libdeflate from git repo (NVIDIA fork with gdeflate)" OFF)
+set(OPENEXR_DEFLATE_REPO "https://github.com/NVIDIA/libdeflate.git" CACHE STRING "Git repo for FetchContent libdeflate source")
+set(OPENEXR_DEFLATE_TAG "gdeflate" CACHE STRING "Git tag/branch for FetchContent libdeflate source")
+
 set (OPENEXR_USE_INTERNAL_DEFLATE OFF)
+set (OPENEXR_USE_FETCHCONTENT_DEFLATE OFF)
 
-if(NOT OPENEXR_FORCE_INTERNAL_DEFLATE)
+if(NOT OPENEXR_FORCE_INTERNAL_DEFLATE AND NOT OPENEXR_FORCE_FETCHCONTENT_DEFLATE)
   #TODO: ^^ Release should not clone from main, this is a place holder
   set(CMAKE_IGNORE_PATH "${CMAKE_CURRENT_BINARY_DIR}/_deps/deflate-src/config;${CMAKE_CURRENT_BINARY_DIR}/_deps/deflate-build/config")
   # First try cmake config
@@ -238,6 +243,50 @@ if(EXR_DEFLATE_LIB)
   message(STATUS "Using externally provided libdeflate: ${EXR_DEFLATE_VERSION}")
   # For OpenEXR.pc.in for static build
   set(EXR_DEFLATE_PKGCONFIG_REQUIRES "libdeflate >= ${EXR_DEFLATE_VERSION}")
+elseif(OPENEXR_FORCE_FETCHCONTENT_DEFLATE)
+  # Using FetchContent to get NVIDIA fork
+  message(STATUS "Fetching libdeflate from ${OPENEXR_DEFLATE_REPO} @ ${OPENEXR_DEFLATE_TAG}")
+
+  include(FetchContent)
+  FetchContent_Declare(Deflate
+    GIT_REPOSITORY ${OPENEXR_DEFLATE_REPO}
+    GIT_TAG ${OPENEXR_DEFLATE_TAG}
+    GIT_SHALLOW ON
+  )
+
+  FetchContent_GetProperties(Deflate)
+  if(NOT Deflate_POPULATED)
+    FetchContent_Populate(Deflate)
+  endif()
+
+  # Build libdeflate as an OBJECT library to avoid symbol conflicts
+  # OBJECT libraries compile source files but don't create a standalone library,
+  # so symbols remain private when linked into OpenEXRCore
+  add_library(deflate_internal OBJECT
+    ${deflate_SOURCE_DIR}/lib/arm/cpu_features.c
+    ${deflate_SOURCE_DIR}/lib/x86/cpu_features.c
+    ${deflate_SOURCE_DIR}/lib/utils.c
+    ${deflate_SOURCE_DIR}/lib/deflate_compress.c
+    ${deflate_SOURCE_DIR}/lib/deflate_decompress.c
+    ${deflate_SOURCE_DIR}/lib/adler32.c
+    ${deflate_SOURCE_DIR}/lib/gdeflate_compress.c
+    ${deflate_SOURCE_DIR}/lib/gdeflate_decompress.c
+    ${deflate_SOURCE_DIR}/lib/zlib_compress.c
+    ${deflate_SOURCE_DIR}/lib/zlib_decompress.c
+  )
+
+  target_include_directories(deflate_internal PUBLIC ${deflate_SOURCE_DIR})
+
+  # Hide all symbols by default to prevent conflicts
+  set_target_properties(deflate_internal PROPERTIES
+    C_VISIBILITY_PRESET hidden
+    VISIBILITY_INLINES_HIDDEN YES
+    POSITION_INDEPENDENT_CODE ON
+  )
+
+  set(EXR_DEFLATE_LIB deflate_internal)
+  set(OPENEXR_USE_FETCHCONTENT_DEFLATE ON)
+  message(STATUS "Using FetchContent libdeflate with gdeflate support")
 else()
   # Using internal deflate
   if(OPENEXR_FORCE_INTERNAL_DEFLATE)
@@ -250,6 +299,41 @@ else()
   set(EXR_DEFLATE_LIB)
 endif()
 
+#######################################
+# Check for gdeflate support in libdeflate
+#######################################
+
+if(OPENEXR_USE_INTERNAL_DEFLATE)
+  # Internal libdeflate currently lacks gdeflate support
+  set(OPENEXR_ENABLE_GDEFLATE OFF)
+  message(STATUS "Using internal libdeflate - gdeflate DISABLED")
+elseif(OPENEXR_USE_FETCHCONTENT_DEFLATE)
+  set(OPENEXR_ENABLE_GDEFLATE ON)
+  message(STATUS "Using FetchContent libdeflate from NVIDIA fork - gdeflate ENABLED")
+else()
+  # Check if external libdeflate has gdeflate support
+  include(CheckCSourceCompiles)
+  set(CMAKE_REQUIRED_LIBRARIES ${EXR_DEFLATE_LIB})
+  check_c_source_compiles("
+    #include <libdeflate.h>
+    int main() {
+      struct libdeflate_gdeflate_compressor* c = 
+        libdeflate_alloc_gdeflate_compressor(1);
+      libdeflate_free_gdeflate_compressor(c);
+      return 0;
+    }
+  " HAVE_LIBDEFLATE_GDEFLATE)
+  set(CMAKE_REQUIRED_LIBRARIES)
+
+  if(HAVE_LIBDEFLATE_GDEFLATE)
+    set(OPENEXR_ENABLE_GDEFLATE ON)
+    message(STATUS "External libdeflate has gdeflate support - ENABLED")
+  else()
+    set(OPENEXR_ENABLE_GDEFLATE OFF)
+    message(STATUS "External libdeflate lacks gdeflate support - DISABLED")
+  endif()
+endif()
+
 
 #######################################
 # Find or download OpenJPH

src/lib/OpenEXR/CMakeLists.txt

@@ -24,6 +24,8 @@ openexr_define_library(OpenEXR
     ImfCompressionAttribute.cpp
     ImfCompressor.cpp
     ImfCompressor.h
+    ImfGdeflateCompressor.cpp
+    ImfGdeflateCompressor.h
     ImfContext.cpp
     ImfContextInit.cpp
     ImfConvert.cpp

src/lib/OpenEXR/ImfCRgbaFile.h

@@ -82,7 +82,8 @@ typedef struct ImfRgba ImfRgba;
 #define IMF_DWAB_COMPRESSION 9
 #define IMF_HTJ2K256_COMPRESSION 10
 #define IMF_HTJ2K32_COMPRESSION 11
-#define IMF_NUM_COMPRESSION_METHODS 12
+#define IMF_GDEFLATE_COMPRESSION 12
+#define IMF_NUM_COMPRESSION_METHODS 13
 
 /*
 ** Channels; values must be the same as in Imf::RgbaChannels.

src/lib/OpenEXR/ImfCompression.cpp

@@ -188,6 +188,12 @@ static const CompressionDesc IdToDesc[] = {
         32,
         true,
         false),
+    CompressionDesc (
+        "gdeflate",
+        "gdeflate compression, in blocks of 16 scan lines.",
+        16,
+        false,
+        false),
 };
 // clang-format on
 
@@ -206,6 +212,7 @@ static const std::map<std::string, Compression> CompressionNameToId = {
     {"dwab", Compression::DWAB_COMPRESSION},
     {"htj2k256", Compression::HTJ2K256_COMPRESSION},
     {"htj2k32", Compression::HTJ2K32_COMPRESSION},
+    {"gdeflate", Compression::GDEFLATE_COMPRESSION},
 };
 
 #define UNKNOWN_COMPRESSION_ID_MSG "INVALID COMPRESSION ID"

src/lib/OpenEXR/ImfCompression.h

@@ -55,6 +55,8 @@ enum IMF_EXPORT_ENUM Compression
 
     HTJ2K32_COMPRESSION = 11,    // High-Throughput JPEG2000 (HTJ2K), 32 scanlines
 
+    GDEFLATE_COMPRESSION = 12,   // gdeflate compression, in blocks of 16 scan lines
+
     NUM_COMPRESSION_METHODS // number of different compression methods
 };
 

src/lib/OpenEXR/ImfCompressor.cpp

@@ -13,6 +13,7 @@
 #include "ImfNamespace.h"
 #include "ImfCompressor.h"
 #include "ImfB44Compressor.h"
+#include "ImfGdeflateCompressor.h"
 #include "ImfDwaCompressor.h"
 #include "ImfPizCompressor.h"
 #include "ImfPxr24Compressor.h"
@@ -344,6 +345,11 @@ newCompressor (Compression c, size_t maxScanLineSize, const Header& hdr)
 
             return new HTCompressor (hdr, static_cast<int> (maxScanLineSize), 32);
 
+        case GDEFLATE_COMPRESSION:
+
+            ret = new GdeflateCompressor (hdr, maxScanLineSize, 16);
+            break;
+
         default: break;
     }
     // clang-format on
@@ -433,6 +439,12 @@ newTileCompressor (
                 static_cast<int> (tileLineSize),
                 static_cast<int> (numTileLines));
 
+        case GDEFLATE_COMPRESSION:
+
+            ret = new GdeflateCompressor (
+                hdr, tileLineSize, static_cast<int> (numTileLines));
+            break;
+
         default: break;
     }
     // clang-format on

src/lib/OpenEXR/ImfGdeflateCompressor.cpp

@@ -0,0 +1,55 @@
+//
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) Contributors to the OpenEXR Project.
+//
+
+//-----------------------------------------------------------------------------
+//
+//  class GdeflateCompressor
+//
+//-----------------------------------------------------------------------------
+
+#include "ImfGdeflateCompressor.h"
+#include "OpenEXRConfigInternal.h"
+
+#ifndef OPENEXR_ENABLE_GDEFLATE
+#include "IexBaseExc.h"
+#endif
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_ENTER
+
+#ifdef OPENEXR_ENABLE_GDEFLATE
+
+GdeflateCompressor::GdeflateCompressor (
+    const Header& hdr, size_t maxScanLineSize, int numScanLines)
+    : Compressor (
+          hdr,
+          EXR_COMPRESSION_GDEFLATE,
+          maxScanLineSize,
+          numScanLines)
+{
+}
+
+GdeflateCompressor::~GdeflateCompressor ()
+{
+}
+
+#else
+
+GdeflateCompressor::GdeflateCompressor (
+    const Header& hdr, size_t maxScanLineSize, int numScanLines)
+    : Compressor (hdr, EXR_COMPRESSION_LAST_TYPE, maxScanLineSize, numScanLines)
+{
+    throw IEX_NAMESPACE::NoImplExc (
+        "Gdeflate support is not enabled in this build of OpenEXR.");
+}
+
+GdeflateCompressor::~GdeflateCompressor ()
+{
+}
+
+#endif
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_SOURCE_EXIT
+
+

src/lib/OpenEXR/ImfGdeflateCompressor.h

@@ -0,0 +1,38 @@
+//
+// SPDX-License-Identifier: BSD-3-Clause
+// Copyright (c) Contributors to the OpenEXR Project.
+//
+
+#ifndef INCLUDED_IMF_GDEFLATE_COMPRESSOR_H
+#define INCLUDED_IMF_GDEFLATE_COMPRESSOR_H
+
+//-----------------------------------------------------------------------------
+//
+//  class GdeflateCompressor -- performs gdeflate compression
+//
+//-----------------------------------------------------------------------------
+
+#include "ImfCompressor.h"
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_ENTER
+
+class GdeflateCompressor : public Compressor
+{
+public:
+    GdeflateCompressor (
+        const Header& hdr, size_t maxScanLineSize, int numScanLines);
+
+    virtual ~GdeflateCompressor ();
+
+    GdeflateCompressor (const GdeflateCompressor& other)            = delete;
+    GdeflateCompressor& operator= (const GdeflateCompressor& other) = delete;
+    GdeflateCompressor (GdeflateCompressor&& other)                 = delete;
+    GdeflateCompressor& operator= (GdeflateCompressor&& other)      = delete;
+};
+
+OPENEXR_IMF_INTERNAL_NAMESPACE_HEADER_EXIT
+
+#endif
+
+
+