Jb/jfr alloc sampler

.github/workflows/dd-sync.yml

@@ -0,0 +1,34 @@
+# .github/workflows/sync-upstream.yml
+name: Sync Upstream
+
+on:
+  schedule:
+    - cron: '22 14 * * *' # Runs every day at 14:15 UTC
+  workflow_dispatch:
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+        fetch-depth: 0
+
+    - name: Pull latest changes from upstream
+      run: |
+        git config --global user.email "sync@datadoghq.com"
+        git config --global user.name "Datadog Syncup Service"
+        git remote add upstream https://github.com/openjdk/jdk.git
+        git fetch upstream
+        git checkout -b upstream-master upstream/master
+        git checkout master
+        git merge upstream-master
+
+    - name: Push changes to downstream
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.GH_PAT }}
+        branch: master

.gitlab-ci.yml

@@ -0,0 +1,43 @@
+image: alpine
+
+stages: [prepare, trigger]
+
+prepare:
+  stage: prepare
+  image: "486234852809.dkr.ecr.us-east-1.amazonaws.com/images/docker:24.0.4-gbi-focal"
+  tags: [ "arch:arm64" ]
+  script:
+    - echo "Preparing to trigger the build pipeline"
+    - echo "FEATURE_VERSION=$(cat make/conf/version-numbers.conf | grep 'DEFAULT_VERSION_FEATURE=' | cut -d '=' -f 2)" > build.env
+  artifacts:
+    reports:
+      dotenv: build.env
+    expire_in: 1 day
+
+build:
+  stage: trigger
+  needs: [prepare]
+  variables:
+    JDK_VERSION: "latest"
+    DEBUG_LEVEL: "fastdebug"
+    HASH: "${CI_COMMIT_SHORT_SHA}"
+  trigger:
+    project: DataDog/openjdk-build
+    strategy: depend
+    branch: main
+    forward:
+      pipeline_variables: true
+
+test:
+  stage: trigger
+  needs: [prepare, build]
+  variables:
+    JDK_VERSION: "${FEATURE_VERSION}"
+    DEBUG_LEVEL: "fastdebug"
+    HASH: "${CI_COMMIT_SHORT_SHA}"
+  trigger:
+    project: DataDog/apm-reliability/async-profiler-build
+    strategy: depend
+    branch: main
+    forward:
+      pipeline_variables: true

src/hotspot/os/posix/dtrace/hotspot_gc.d

@@ -0,0 +1,9 @@
+provider hotspot_gc {
+  probe AllocObject__sample(char*, size_t, size_t);
+};
+
+#pragma D attributes Standard/Standard/Common provider hotspot_gc provider
+#pragma D attributes Private/Private/Unknown provider hotspot_gc module
+#pragma D attributes Private/Private/Unknown provider hotspot_gc function
+#pragma D attributes Standard/Standard/Common provider hotspot_gc name
+#pragma D attributes Evolving/Evolving/Common provider hotspot_gc args

src/hotspot/share/gc/shared/allocTracer.cpp

@@ -32,7 +32,6 @@
 #endif
 
 void AllocTracer::send_allocation_outside_tlab(Klass* klass, HeapWord* obj, size_t alloc_size, JavaThread* thread) {
-  JFR_ONLY(JfrAllocationTracer tracer(klass, obj, alloc_size, true, thread);)
   EventObjectAllocationOutsideTLAB event;
   if (event.should_commit()) {
     event.set_objectClass(klass);
@@ -42,7 +41,6 @@ void AllocTracer::send_allocation_outside_tlab(Klass* klass, HeapWord* obj, size
 }
 
 void AllocTracer::send_allocation_in_new_tlab(Klass* klass, HeapWord* obj, size_t tlab_size, size_t alloc_size, JavaThread* thread) {
-  JFR_ONLY(JfrAllocationTracer tracer(klass, obj, alloc_size, false, thread);)
   EventObjectAllocationInNewTLAB event;
   if (event.should_commit()) {
     event.set_objectClass(klass);
@@ -52,6 +50,10 @@ void AllocTracer::send_allocation_in_new_tlab(Klass* klass, HeapWord* obj, size_
   }
 }
 
+void AllocTracer::send_allocation_sample(Klass* klass, HeapWord* obj, size_t alloc_size, size_t weight, bool large_allocation, JavaThread* thread) {
+  JFR_ONLY(JfrAllocationTracer tracer(klass, obj, alloc_size, large_allocation, thread);)
+}
+
 void AllocTracer::send_allocation_requiring_gc_event(size_t size, uint gcId) {
   EventAllocationRequiringGC event;
   if (event.should_commit()) {

src/hotspot/share/gc/shared/allocTracer.hpp

@@ -32,6 +32,7 @@ class AllocTracer : AllStatic {
   public:
     static void send_allocation_outside_tlab(Klass* klass, HeapWord* obj, size_t alloc_size, JavaThread* thread);
     static void send_allocation_in_new_tlab(Klass* klass, HeapWord* obj, size_t tlab_size, size_t alloc_size, JavaThread* thread);
+    static void send_allocation_sample(Klass* klass, HeapWord* obj, size_t alloc_size, size_t weight, bool large_allocation, JavaThread* thread);
     static void send_allocation_requiring_gc_event(size_t size, uint gcId);
 };
 

src/hotspot/share/gc/shared/memAllocator.cpp

@@ -30,6 +30,7 @@
 #include "gc/shared/memAllocator.hpp"
 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
 #include "gc/shared/tlab_globals.hpp"
+#include "jfr/jfrEvents.hpp"
 #include "memory/universe.hpp"
 #include "oops/arrayOop.hpp"
 #include "oops/oop.inline.hpp"
@@ -41,6 +42,7 @@
 #include "services/lowMemoryDetector.hpp"
 #include "utilities/align.hpp"
 #include "utilities/copy.hpp"
+#include "utilities/dtrace.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 class MemAllocator::Allocation: StackObj {
@@ -171,7 +173,8 @@ void MemAllocator::Allocation::notify_allocation_jvmti_sampler() {
     return;
   }
 
-  if (!_allocated_outside_tlab && _allocated_tlab_size == 0 && !_tlab_end_reset_for_sample) {
+  bool hit_mark = _allocated_tlab_size != 0 || _tlab_end_reset_for_sample;
+  if (!_allocated_outside_tlab && !hit_mark) {
     // Sample if it's a non-TLAB allocation, or a TLAB allocation that either refills the TLAB
     // or expands it due to taking a sampler induced slow path.
     return;
@@ -181,23 +184,21 @@ void MemAllocator::Allocation::notify_allocation_jvmti_sampler() {
   // before doing the callback. The callback is done in the destructor of
   // the JvmtiSampledObjectAllocEventCollector.
   size_t bytes_since_last = 0;
-
+  size_t bytes_since_allocation = 0;
   {
     PreserveObj obj_h(_thread, _obj_ptr);
     JvmtiSampledObjectAllocEventCollector collector;
     size_t size_in_bytes = _allocator._word_size * HeapWordSize;
-    ThreadLocalAllocBuffer& tlab = _thread->tlab();
 
-    if (!_allocated_outside_tlab) {
-      bytes_since_last = tlab.bytes_since_last_sample_point();
+    if (_thread->heap_samplers().jvmti().check_for_sampling(&bytes_since_allocation, size_in_bytes, !_allocated_outside_tlab)) {
+      JvmtiExport::sampled_object_alloc_event_collector(obj_h());
     }
-
-    _thread->heap_sampler().check_for_sampling(obj_h(), size_in_bytes, bytes_since_last);
   }
 
-  if (_tlab_end_reset_for_sample || _allocated_tlab_size != 0) {
+  if (hit_mark) {
+    ThreadHeapSampler& sampler = _thread->heap_samplers().jvmti();
     // Tell tlab to forget bytes_since_last if we passed it to the heap sampler.
-    _thread->tlab().set_sample_end(bytes_since_last != 0);
+    sampler.update_bytes(_thread->tlab().set_sample_end(sampler.bytes_until_sample()), !_allocated_outside_tlab);
   }
 }
 
@@ -217,6 +218,36 @@ void MemAllocator::Allocation::notify_allocation_jfr_sampler() {
     AllocTracer::send_allocation_in_new_tlab(obj()->klass(), mem, _allocated_tlab_size * HeapWordSize,
                                              size_in_bytes, _thread);
   }
+
+  EventObjectAllocationSample event;
+  if (!event.should_commit()) {
+    return;
+  }
+
+  bool hit_mark = _allocated_tlab_size != 0 || _tlab_end_reset_for_sample;
+  if (!_allocated_outside_tlab && !hit_mark) {
+    // Sample if it's a non-TLAB allocation, or a TLAB allocation that either refills the TLAB
+    // or expands it due to taking a sampler induced slow path.
+    return;
+  }
+
+  ThreadHeapSampler& sampler = _thread->heap_samplers().jfr();
+  if (sampler.bytes_until_sample() == static_cast<size_t>(-1)) {
+    return;
+  }
+
+  size_t bytes_since_allocation = 0;
+  if (sampler.check_for_sampling(&bytes_since_allocation, size_in_bytes, !_allocated_outside_tlab)) {
+    size_t weight = bytes_since_allocation == 0 ? size_in_bytes : bytes_since_allocation;
+    AllocTracer::send_allocation_sample(obj()->klass(), mem, size_in_bytes, weight, _allocated_outside_tlab, _thread);
+    HOTSPOT_GC_ALLOCOBJECT_SAMPLE(obj()->klass()->name()->as_C_string(), size_in_bytes, weight);
+  }
+
+  if (hit_mark) {
+    // Tell tlab to forget bytes_since_last if we passed it to the heap sampler.
+    size_t bytes_inc = _thread->tlab().set_sample_end(sampler.bytes_until_sample());
+    sampler.update_bytes(bytes_inc, !_allocated_outside_tlab);
+  }
 }
 
 void MemAllocator::Allocation::notify_allocation_dtrace_sampler() {
@@ -258,7 +289,9 @@ HeapWord* MemAllocator::mem_allocate_inside_tlab_slow(Allocation& allocation) co
   HeapWord* mem = nullptr;
   ThreadLocalAllocBuffer& tlab = _thread->tlab();
 
-  if (JvmtiExport::should_post_sampled_object_alloc()) {
+  EventObjectAllocationSample event;
+
+  if (JvmtiExport::should_post_sampled_object_alloc() || event.should_commit()) {
     tlab.set_back_allocation_end();
     mem = tlab.allocate(_word_size);
 

src/hotspot/share/gc/shared/threadLocalAllocBuffer.cpp

@@ -50,7 +50,6 @@ ThreadLocalAllocBuffer::ThreadLocalAllocBuffer() :
   _desired_size(0),
   _refill_waste_limit(0),
   _allocated_before_last_gc(0),
-  _bytes_since_last_sample_point(0),
   _number_of_refills(0),
   _refill_waste(0),
   _gc_waste(0),
@@ -203,6 +202,7 @@ void ThreadLocalAllocBuffer::initialize(HeapWord* start,
   set_end(end);
   set_allocation_end(end);
   invariants();
+  _end_backup = nullptr;
 }
 
 void ThreadLocalAllocBuffer::initialize() {
@@ -313,21 +313,27 @@ void ThreadLocalAllocBuffer::print_stats(const char* tag) {
             _refill_waste * HeapWordSize);
 }
 
-void ThreadLocalAllocBuffer::set_sample_end(bool reset_byte_accumulation) {
+size_t ThreadLocalAllocBuffer::set_sample_end(size_t bytes_until_sample) {
   size_t heap_words_remaining = pointer_delta(_end, _top);
-  size_t bytes_until_sample = thread()->heap_sampler().bytes_until_sample();
   size_t words_until_sample = bytes_until_sample / HeapWordSize;
 
-  if (reset_byte_accumulation) {
-    _bytes_since_last_sample_point = 0;
-  }
-
   if (heap_words_remaining > words_until_sample) {
     HeapWord* new_end = _top + words_until_sample;
+    if (_end_backup != nullptr) {
+      if (new_end > _end_backup) {
+        HeapWord* tmp = new_end;
+        new_end = _end_backup;
+        _end_backup = tmp;
+      }
+    }
     set_end(new_end);
-    _bytes_since_last_sample_point += bytes_until_sample;
+    return bytes_until_sample;
   } else {
-    _bytes_since_last_sample_point += heap_words_remaining * HeapWordSize;
+    if (_end_backup != nullptr) {
+      set_end(_end_backup);
+      _end_backup = nullptr;
+    }
+    return heap_words_remaining * HeapWordSize;
   }
 }
 

src/hotspot/share/gc/shared/threadLocalAllocBuffer.hpp

@@ -51,12 +51,12 @@ class ThreadLocalAllocBuffer: public CHeapObj<mtThread> {
   HeapWord* _top;                                // address after last allocation
   HeapWord* _pf_top;                             // allocation prefetch watermark
   HeapWord* _end;                                // allocation end (can be the sampling end point or _allocation_end)
+  HeapWord* _end_backup;                         // backup of _end for interleaved jvmti and jfr sampling
   HeapWord* _allocation_end;                     // end for allocations (actual TLAB end, excluding alignment_reserve)
 
   size_t    _desired_size;                       // desired size   (including alignment_reserve)
   size_t    _refill_waste_limit;                 // hold onto tlab if free() is larger than this
   size_t    _allocated_before_last_gc;           // total bytes allocated up until the last gc
-  size_t    _bytes_since_last_sample_point;      // bytes since last sample point.
 
   static size_t   _max_size;                          // maximum size of any TLAB
   static int      _reserve_for_allocation_prefetch;   // Reserve at the end of the TLAB
@@ -124,7 +124,6 @@ class ThreadLocalAllocBuffer: public CHeapObj<mtThread> {
   size_t free() const                            { return pointer_delta(end(), top()); }
   // Don't discard tlab if remaining space is larger than this.
   size_t refill_waste_limit() const              { return _refill_waste_limit; }
-  size_t bytes_since_last_sample_point() const   { return _bytes_since_last_sample_point; }
 
   // For external inspection.
   const HeapWord* start_relaxed() const;
@@ -168,7 +167,7 @@ class ThreadLocalAllocBuffer: public CHeapObj<mtThread> {
   void initialize();
 
   void set_back_allocation_end();
-  void set_sample_end(bool reset_byte_accumulation);
+  size_t set_sample_end(size_t bytes_until_sample);
 
   static size_t refill_waste_limit_increment();
 

src/hotspot/share/jfr/recorder/service/jfrEventThrottler.cpp

@@ -27,6 +27,7 @@
 #include "jfr/recorder/service/jfrEventThrottler.hpp"
 #include "jfr/utilities/jfrSpinlockHelper.hpp"
 #include "logging/log.hpp"
+#include "runtime/threadHeapSampler.hpp"
 
 constexpr static const JfrSamplerParams _disabled_params = {
                                                              0, // sample points per window
@@ -72,6 +73,9 @@ void JfrEventThrottler::configure(JfrEventId event_id, int64_t sample_size, int6
   }
   assert(_throttler != nullptr, "JfrEventThrottler has not been properly initialized");
   _throttler->configure(sample_size, period_ms);
+
+  // TODO: Hack to get the allocation sampler going
+  ThreadHeapSamplers::set_jfr_sampling_interval(512 * 1024);
 }
 
 /*

src/hotspot/share/prims/jvmtiEnv.cpp

@@ -3760,7 +3760,7 @@ JvmtiEnv::SetHeapSamplingInterval(jint sampling_interval) {
   if (sampling_interval < 0) {
     return JVMTI_ERROR_ILLEGAL_ARGUMENT;
   }
-  ThreadHeapSampler::set_sampling_interval(sampling_interval);
+  ThreadHeapSamplers::set_jvmti_sampling_interval(sampling_interval);
   return JVMTI_ERROR_NONE;
 } /* end SetHeapSamplingInterval */
 

src/hotspot/share/runtime/thread.hpp

@@ -267,7 +267,7 @@ class Thread: public ThreadShadow {
   ThreadLocalAllocBuffer _tlab;                 // Thread-local eden
   jlong _allocated_bytes;                       // Cumulative number of bytes allocated on
                                                 // the Java heap
-  ThreadHeapSampler _heap_sampler;              // For use when sampling the memory.
+  ThreadHeapSamplers _heap_samplers;            // Used by the JVMTI allocation sampler
 
   ThreadStatisticalInfo _statistical_info;      // Statistics about the thread
 
@@ -418,7 +418,7 @@ class Thread: public ThreadShadow {
   void incr_allocated_bytes(jlong size) { _allocated_bytes += size; }
   inline jlong cooked_allocated_bytes();
 
-  ThreadHeapSampler& heap_sampler()     { return _heap_sampler; }
+  ThreadHeapSamplers& heap_samplers()  { return _heap_samplers; }
 
   ThreadStatisticalInfo& statistical_info() { return _statistical_info; }