diff --git a/pp/bare_bones/bit_sequence.h b/pp/bare_bones/bit_sequence.h index 7ba0f24888..c62608cde2 100644 --- a/pp/bare_bones/bit_sequence.h +++ b/pp/bare_bones/bit_sequence.h @@ -170,29 +170,32 @@ class BitSequenceReader { uint64_t size_; }; +#pragma pack(push, 1) + template requires std::is_same_v -class PROMPP_ATTRIBUTE_PACKED CompactBitSequenceBase { +class CompactBitSequenceBase { public: + using SharedPtr = BareBones::SharedPtr; + CompactBitSequenceBase() = default; - CompactBitSequenceBase(const CompactBitSequenceBase& other) - : memory_(static_cast(std::malloc(other.allocated_memory()))), - size_in_bits_(other.size_in_bits_), - allocation_size_index_(other.allocation_size_index_) { - std::memcpy(memory_, other.memory_, other.allocated_memory()); + PROMPP_ALWAYS_INLINE CompactBitSequenceBase(const CompactBitSequenceBase& other) + : memory_(other.stream_allocated_memory(), 0), size_in_bits_(other.size_in_bits_), allocation_size_index_(other.allocation_size_index_) { + std::memcpy(memory_.get(), other.memory_.get(), other.stream_allocated_memory()); } PROMPP_ALWAYS_INLINE CompactBitSequenceBase(CompactBitSequenceBase&& other) noexcept - : memory_(other.memory_), size_in_bits_(other.size_in_bits_), allocation_size_index_(std::exchange(other.allocation_size_index_, 0)) { - other.memory_ = nullptr; + : memory_(std::move(other.memory_)), + size_in_bits_(std::exchange(other.size_in_bits_, 0)), + allocation_size_index_(std::exchange(other.allocation_size_index_, 0)) { other.size_in_bits_ = 0; } - CompactBitSequenceBase& operator=(const CompactBitSequenceBase& other) { - if (this != &other) { - std::free(memory_); + PROMPP_ALWAYS_INLINE CompactBitSequenceBase& operator=(const CompactBitSequenceBase& other) { + if (this != &other) [[likely]] { + const auto size = other.stream_allocated_memory(); + memory_.reallocate(0, size); + std::memcpy(memory_.get(), other.memory_.get(), size); - memory_ = static_cast(std::malloc(other.allocated_memory())); - std::memcpy(memory_, other.memory_, other.allocated_memory()); size_in_bits_ = other.size_in_bits_; allocation_size_index_ = other.allocation_size_index_; } @@ -200,36 +203,24 @@ class PROMPP_ATTRIBUTE_PACKED CompactBitSequenceBase { return *this; } - CompactBitSequenceBase& operator=(CompactBitSequenceBase&& other) noexcept { - if (this != &other) { - std::free(memory_); - - memory_ = other.memory_; - other.memory_ = nullptr; - - size_in_bits_ = other.size_in_bits_; - other.size_in_bits_ = 0; - + PROMPP_ALWAYS_INLINE CompactBitSequenceBase& operator=(CompactBitSequenceBase&& other) noexcept { + if (this != &other) [[likely]] { + memory_ = std::move(other.memory_); + size_in_bits_ = std::exchange(other.size_in_bits_, 0); allocation_size_index_ = std::exchange(other.allocation_size_index_, 0); } return *this; } - ~CompactBitSequenceBase() { - std::free(memory_); - memory_ = nullptr; - } - PROMPP_ALWAYS_INLINE bool operator==(const CompactBitSequenceBase& other) const noexcept { - return size_in_bits_ == other.size_in_bits_ && memcmp(memory_, other.memory_, size_in_bytes()) == 0; + return size_in_bits_ == other.size_in_bits_ && std::memcmp(memory_.get(), other.memory_.get(), size_in_bytes()) == 0; } PROMPP_ALWAYS_INLINE void clear() noexcept { + memory_.clear(); size_in_bits_ = 0; allocation_size_index_ = 0; - std::free(memory_); - memory_ = nullptr; } [[nodiscard]] static consteval uint32_t reserved_size_in_bits() noexcept { return kReservedSizeBits; } @@ -240,93 +231,112 @@ class PROMPP_ATTRIBUTE_PACKED CompactBitSequenceBase { } [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size_in_bits() const noexcept { return size_in_bits_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size_in_bytes() const noexcept { return Bit::to_ceil_bytes(size_in_bits_); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size_in_bytes() const noexcept { return Bit::to_ceil_bytes(size_in_bits()); } [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t allocated_memory() const noexcept { - return is_read_only() ? (size_in_bytes() + Bit::to_bytes(kReservedSizeBits)) : kAllocationSizesTable[allocation_size_index_].bytes(); + return stream_allocated_memory() + (memory_.get() != nullptr ? SharedPtr::kControlBlockSize : 0); } [[nodiscard]] PROMPP_ALWAYS_INLINE uint8_t allocation_size_index() const noexcept { return allocation_size_index_; } [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_read_only() const noexcept { return allocation_size_index_ == kNoAllocationIndex; } - [[nodiscard]] PROMPP_ALWAYS_INLINE std::span filled_bytes() const noexcept { return {memory_, Bit::to_bytes(size_in_bits_)}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE std::span filled_bytes() const noexcept { return {memory_.get(), Bit::to_bytes(size_in_bits())}; } template [[nodiscard]] PROMPP_ALWAYS_INLINE std::span bytes() const noexcept { - return {reinterpret_cast(memory_), size_in_bytes() / sizeof(T)}; + return {reinterpret_cast(memory_.get()), size_in_bytes() / sizeof(T)}; } [[nodiscard]] PROMPP_ALWAYS_INLINE std::span bytes() const noexcept { return bytes(); } - [[nodiscard]] PROMPP_ALWAYS_INLINE const uint8_t* raw_bytes() const noexcept { return memory_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint8_t* raw_bytes() noexcept { return memory_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const uint8_t* raw_bytes() const noexcept { return memory_.get(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint8_t* raw_bytes() noexcept { return memory_.get(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE SharedPtr shared_memory() const noexcept { return memory_; } PROMPP_ALWAYS_INLINE void shrink_to_fit() noexcept { - memory_ = static_cast(std::realloc(memory_, size_in_bytes() + Bit::to_bytes(kReservedSizeBits))); + const auto size = Bit::to_bytes(size_in_bits()) + Bit::to_bytes(kReservedSizeBits); + memory_.reallocate(size, size); allocation_size_index_ = kNoAllocationIndex; } PROMPP_ALWAYS_INLINE void rewind() noexcept { - if (size_in_bits_ > 0) [[likely]] { - memset(memory_, '\0', allocated_memory()); - size_in_bits_ = 0; + if (size_in_bits() == 0) [[unlikely]] { + return; + } + + if (memory_.non_atomic_is_unique()) [[likely]] { + memset(memory_.get(), '\0', stream_allocated_memory()); + } else { + memory_.clear(); } + + size_in_bits_ = 0; } protected: static constexpr uint32_t kNoAllocationIndex = std::numeric_limits::max(); - uint8_t* memory_{}; + SharedPtr memory_; uint32_t size_in_bits_{}; uint8_t allocation_size_index_{}; - void reserve_enough_memory_if_needed() noexcept { + PROMPP_ALWAYS_INLINE void reserve_enough_memory_if_needed() noexcept { assert(!is_read_only()); const auto old_size = kAllocationSizesTable[allocation_size_index_]; - if (size_in_bits_ + kReservedSizeBits > old_size.bits) [[unlikely]] { + if (size_in_bits() + kReservedSizeBits > old_size.bits) [[unlikely]] { ++allocation_size_index_; - assert(allocation_size_index_ < std::size(kAllocationSizesTable)); - - auto new_size = kAllocationSizesTable[allocation_size_index_].bytes(); - memory_ = reinterpret_cast(std::realloc(memory_, new_size)); - std::memset(memory_ + old_size.bytes(), 0, new_size - old_size.bytes()); + reallocate(old_size.bytes()); } } - void reserve_enough_memory_if_needed(uint32_t needed_size) noexcept { + PROMPP_ALWAYS_INLINE void reserve_enough_memory_if_needed(uint32_t needed_size) noexcept { assert(!is_read_only()); - needed_size += size_in_bits_ + kReservedSizeBits; + needed_size += size_in_bits() + kReservedSizeBits; auto new_allocation_size_index = allocation_size_index_; while (needed_size > kAllocationSizesTable[new_allocation_size_index]) { ++new_allocation_size_index; } - if (new_allocation_size_index > allocation_size_index_) { + if (new_allocation_size_index > allocation_size_index_) [[unlikely]] { const auto old_size = kAllocationSizesTable[allocation_size_index_]; allocation_size_index_ = new_allocation_size_index; - assert(new_allocation_size_index < std::size(kAllocationSizesTable)); - - auto new_size = kAllocationSizesTable[allocation_size_index_].bytes(); - memory_ = reinterpret_cast(std::realloc(memory_, new_size)); - std::memset(memory_ + old_size.bytes(), 0, new_size - old_size.bytes()); + reallocate(old_size.bytes()); } } + PROMPP_ALWAYS_INLINE void reallocate(const uint32_t old_size_bytes) { + assert(allocation_size_index_ < std::size(kAllocationSizesTable)); + + const auto new_size = kAllocationSizesTable[allocation_size_index_].bytes(); + memory_.reallocate(old_size_bytes, new_size); + std::memset(memory_.get() + old_size_bytes, 0, new_size - old_size_bytes); + } + template [[nodiscard]] PROMPP_ALWAYS_INLINE T* unfilled_memory() const noexcept { - return reinterpret_cast(memory_ + Bit::to_bytes(size_in_bits_)); + return reinterpret_cast(memory_.get() + Bit::to_bytes(size_in_bits())); } - [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t unfilled_bits_in_byte() const noexcept { return size_in_bits_ % Bit::kByteBits; } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t unfilled_bits_in_byte() const noexcept { return size_in_bits() % Bit::kByteBits; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t stream_allocated_memory() const noexcept { + return is_read_only() ? (size_in_bytes() + Bit::to_bytes(kReservedSizeBits)) : kAllocationSizesTable[allocation_size_index_].bytes(); + } }; +#pragma pack(pop) + template requires std::is_same_v class PROMPP_ATTRIBUTE_PACKED CompactBitSequence : public CompactBitSequenceBase { public: - [[nodiscard]] PROMPP_ALWAYS_INLINE BitSequenceReader reader() const noexcept { return {Base::memory_, size_in_bits_}; }; + using Base = CompactBitSequenceBase; + using Base::size_in_bits; + using Base::size_in_bits_; + + [[nodiscard]] PROMPP_ALWAYS_INLINE BitSequenceReader reader() const noexcept { return {Base::memory_.get(), size_in_bits()}; }; PROMPP_ALWAYS_INLINE void trim_lower_bytes(uint32_t bytes_count) { assert(Bit::to_bits(bytes_count) <= Base::size_in_bits()); const size_t new_size_in_bytes = Base::size_in_bytes() - bytes_count; - memmove(Base::memory_, Base::memory_ + bytes_count, new_size_in_bytes); - memset(Base::memory_ + new_size_in_bytes, '\0', bytes_count); + memmove(Base::memory_.get(), Base::memory_.get() + bytes_count, new_size_in_bytes); + memset(Base::memory_.get() + new_size_in_bytes, '\0', bytes_count); size_in_bits_ -= Bit::to_bits(bytes_count); } @@ -399,7 +409,7 @@ class PROMPP_ATTRIBUTE_PACKED CompactBitSequence : public CompactBitSequenceBase uint32_t count = Bit::to_ceil_bytes(bit_count); if (unfilled_bits_in_byte() == 0) [[unlikely]] { - std::memcpy(Base::memory_ + Base::size_in_bytes(), bytes, count); + std::memcpy(Base::memory_.get() + Base::size_in_bytes(), bytes, count); size_in_bits_ += bit_count; } else { for (; count >= sizeof(uint64_t); count -= sizeof(uint64_t), bytes += sizeof(uint64_t)) { @@ -419,10 +429,7 @@ class PROMPP_ATTRIBUTE_PACKED CompactBitSequence : public CompactBitSequenceBase PROMPP_ALWAYS_INLINE void push_back_bytes(std::span bytes) noexcept { push_back_bytes(bytes.data(), Bit::to_bits(bytes.size())); } private: - using Base = CompactBitSequenceBase; - using Base::reserve_enough_memory_if_needed; - using Base::size_in_bits_; using Base::unfilled_bits_in_byte; using Base::unfilled_memory; }; @@ -434,8 +441,7 @@ class BitSequence { Memory data_; PROMPP_ALWAYS_INLINE void reserve_enough_memory_if_needed() noexcept { - if (size_ >= max_size_for_current_data_size_) { - [[unlikely]]; + if (size_ >= max_size_for_current_data_size_) [[unlikely]] { reserve_memory(size_); } } diff --git a/pp/bare_bones/memory.h b/pp/bare_bones/memory.h index 2002d22357..74de4538c9 100644 --- a/pp/bare_bones/memory.h +++ b/pp/bare_bones/memory.h @@ -197,29 +197,74 @@ struct DefaultReallocator { PROMPP_ALWAYS_INLINE static void free(void* memory) { return std::free(memory); } }; -template -class SharedPtr { +template +concept SharedPtrControlBlockInterface = requires(ControlBlock control_block, const ControlBlock const_control_block) { + requires std::integral; + requires std::integral; + + { control_block.ref_count() } -> std::same_as; + { const_control_block.ref_count() } -> std::same_as; + { control_block.atomic_ref_count() } -> std::same_as; + + { control_block.items_count() } -> std::same_as; + { control_block.set_items_count(typename ControlBlock::ItemCounter()) }; +}; + +class SharedPtrControlBlockWithItemCount { public: using RefCounter = uint32_t; using ItemCounter = uint32_t; using AtomicRefCounter = std::atomic_ref; - struct ControlBlock { - RefCounter ref_count{1}; - ItemCounter constructed_item_count{}; + [[nodiscard]] PROMPP_ALWAYS_INLINE ItemCounter items_count() const noexcept { return items_count_; } + PROMPP_ALWAYS_INLINE void set_items_count(ItemCounter count) noexcept { items_count_ = count; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE RefCounter& ref_count() noexcept { return ref_count_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE RefCounter ref_count() const noexcept { return ref_count_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE AtomicRefCounter atomic_ref_count() noexcept { return AtomicRefCounter(ref_count_); } + + private: + RefCounter ref_count_{1}; + ItemCounter items_count_{}; +}; +static_assert(SharedPtrControlBlockInterface); + +class SharedPtrControlBlock { + public: + using RefCounter = uint32_t; + using ItemCounter = uint32_t; + using AtomicRefCounter = std::atomic_ref; - [[nodiscard]] PROMPP_ALWAYS_INLINE AtomicRefCounter atomic_ref_count() noexcept { return AtomicRefCounter(ref_count); } - }; + [[nodiscard]] PROMPP_ALWAYS_INLINE static ItemCounter items_count() noexcept { return 0; } + PROMPP_ALWAYS_INLINE static void set_items_count(ItemCounter) noexcept {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE RefCounter& ref_count() noexcept { return ref_count_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE RefCounter ref_count() const noexcept { return ref_count_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE AtomicRefCounter atomic_ref_count() noexcept { return AtomicRefCounter(ref_count_); } + + private: + RefCounter ref_count_{1}; +}; + +static_assert(SharedPtrControlBlockInterface); + +template +class SharedPtr { + public: + using ControlBlock = ControlBlockType; + + static_assert(IsTriviallyReallocatable::value); + static_assert(IsTriviallyDestructible::value); static constexpr uint32_t kControlBlockSize = sizeof(ControlBlock); SharedPtr() = default; - explicit PROMPP_ALWAYS_INLINE SharedPtr(uint32_t size, ItemCounter constructed_item_count = 0) : data_(nullptr) { + PROMPP_ALWAYS_INLINE SharedPtr(uint32_t size, ControlBlock::ItemCounter items_count) : data_(nullptr) { non_atomic_reallocate(size); - set_constructed_item_count(constructed_item_count); + set_items_count(items_count); } PROMPP_ALWAYS_INLINE SharedPtr(const SharedPtr& other) noexcept : data_(other.data_) { inc_ref_counter(); } - SharedPtr(SharedPtr&& other) noexcept : data_(std::exchange(other.data_, nullptr)) {} + PROMPP_ALWAYS_INLINE SharedPtr(SharedPtr&& other) noexcept : data_(std::exchange(other.data_, nullptr)) {} PROMPP_ALWAYS_INLINE ~SharedPtr() { dec_ref_counter(); } @@ -242,26 +287,27 @@ class SharedPtr { return *this; } - PROMPP_ALWAYS_INLINE friend void swap(SharedPtr& a, SharedPtr& b) noexcept { std::swap(a.data_, b.data_); } - - PROMPP_ALWAYS_INLINE void non_atomic_reallocate(uint32_t size) noexcept { - PRAGMA_DIAGNOSTIC(push) - PRAGMA_DIAGNOSTIC(ignored DIAGNOSTIC_CLASS_MEMACCESS) - auto control_block = static_cast(Reallocator::reallocate(raw_memory(), kControlBlockSize + size * sizeof(T))); - PRAGMA_DIAGNOSTIC(pop) - - if (data_ == nullptr) [[likely]] { - std::construct_at(control_block); + PROMPP_ALWAYS_INLINE void reallocate(uint32_t old_size, uint32_t new_size) { + if (non_atomic_is_unique()) [[likely]] { + non_atomic_reallocate(new_size); } else { - control_block->ref_count = 1; + const SharedPtr old(std::move(*this)); + + // NOLINTNEXTLINE(clang-analyzer-cplusplus.Move) + non_atomic_reallocate(new_size); + set_items_count(old.items_count()); + if (old_size > 0) [[likely]] { + PRAGMA_DIAGNOSTIC(push) + PRAGMA_DIAGNOSTIC(ignored DIAGNOSTIC_CLASS_MEMACCESS) + std::memcpy(data_, old.get(), old_size * sizeof(T)); + PRAGMA_DIAGNOSTIC(pop) + } } - - data_ = reinterpret_cast(control_block + 1); } - [[nodiscard]] PROMPP_ALWAYS_INLINE RefCounter non_atomic_ref_count() const noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE ControlBlock::RefCounter non_atomic_ref_count() const noexcept { if (auto block = control_block(); block != nullptr) [[likely]] { - return block->ref_count; + return block->ref_count(); } return 0; @@ -269,33 +315,53 @@ class SharedPtr { [[nodiscard]] PROMPP_ALWAYS_INLINE bool non_atomic_is_unique() const noexcept { if (auto block = control_block(); block != nullptr) [[likely]] { - return block->ref_count == 1; + return block->ref_count() == 1; } return true; } - [[nodiscard]] PROMPP_ALWAYS_INLINE ItemCounter constructed_item_count() const noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE ControlBlock::ItemCounter items_count() const noexcept { if (auto block = control_block(); block != nullptr) [[likely]] { - return block->constructed_item_count; + return block->items_count(); } return 0; } - PROMPP_ALWAYS_INLINE void set_constructed_item_count(ItemCounter count) noexcept { + PROMPP_ALWAYS_INLINE void set_items_count(ControlBlock::ItemCounter count) noexcept { if (auto block = control_block(); block != nullptr) [[likely]] { - block->constructed_item_count = count; + block->set_items_count(count); } } [[nodiscard]] PROMPP_ALWAYS_INLINE T* get() const noexcept { return data_; } - PROMPP_ALWAYS_INLINE void swap(SharedPtr& other) noexcept { std::swap(data_, other.data_); } + PROMPP_ALWAYS_INLINE void clear() noexcept { + dec_ref_counter(); + data_ = nullptr; + } private: T* data_{nullptr}; + PROMPP_ALWAYS_INLINE void non_atomic_reallocate(uint32_t size) noexcept { + PRAGMA_DIAGNOSTIC(push) + PRAGMA_DIAGNOSTIC(ignored DIAGNOSTIC_CLASS_MEMACCESS) + auto control_block = static_cast(Reallocator::reallocate(raw_memory(), kControlBlockSize + size * sizeof(T))); + PRAGMA_DIAGNOSTIC(pop) + + if (control_block == nullptr) [[unlikely]] { + std::abort(); + } + + if (data_ == nullptr) { + std::construct_at(control_block); + } + + data_ = reinterpret_cast(control_block + 1); + } + PROMPP_ALWAYS_INLINE void inc_ref_counter() noexcept { if (auto block = control_block(); block != nullptr) [[likely]] { ++block->atomic_ref_count(); @@ -304,26 +370,21 @@ class SharedPtr { PROMPP_ALWAYS_INLINE void dec_ref_counter() noexcept { if (auto block = control_block(); block != nullptr) [[likely]] { - if (block->ref_count == 1) [[likely]] { + if (block->ref_count() == 1) [[likely]] { destroy(); } else { - --block->atomic_ref_count(); + if (--block->atomic_ref_count() == 0) [[unlikely]] { + destroy(); + } } } } PROMPP_ALWAYS_INLINE void destroy() noexcept { - destroy_constructed_items(); Reallocator::free(raw_memory()); data_ = nullptr; } - PROMPP_ALWAYS_INLINE void destroy_constructed_items() noexcept { - for (T *it = reinterpret_cast(data_), *end = it + control_block()->constructed_item_count; it != end; ++it) { - std::destroy_at(it); - } - } - [[nodiscard]] PROMPP_ALWAYS_INLINE ControlBlock* control_block() noexcept { return static_cast(raw_memory()); } [[nodiscard]] PROMPP_ALWAYS_INLINE const ControlBlock* control_block() const noexcept { return static_cast(raw_memory()); } @@ -334,7 +395,7 @@ template class SharedMemory : public GenericMemory, uint32_t, T> { public: using SizeType = uint32_t; - using SharedPtr = BareBones::SharedPtr; + using SharedPtr = BareBones::SharedPtr; SharedMemory() = default; SharedMemory(const SharedMemory&) = default; @@ -350,8 +411,8 @@ class SharedMemory : public GenericMemory, uint32_t return *this; } - [[nodiscard]] PROMPP_ALWAYS_INLINE typename SharedPtr::ItemCounter constructed_item_count() const noexcept { return data_.constructed_item_count(); } - PROMPP_ALWAYS_INLINE void set_constructed_item_count(typename SharedPtr::ItemCounter count) noexcept { data_.set_constructed_item_count(count); } + [[nodiscard]] PROMPP_ALWAYS_INLINE SharedPtr::ControlBlock::ItemCounter items_count() const noexcept { return data_.items_count(); } + PROMPP_ALWAYS_INLINE void set_items_count(SharedPtr::ControlBlock::ItemCounter count) noexcept { data_.set_items_count(count); } [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return size_ * sizeof(T) + (data_.get() != nullptr ? sizeof(SharedPtr::kControlBlockSize) : 0); @@ -367,17 +428,7 @@ class SharedMemory : public GenericMemory, uint32_t [[nodiscard]] PROMPP_ALWAYS_INLINE const T* data() const noexcept { return data_.get(); } PROMPP_ALWAYS_INLINE void resize(SizeType new_size) noexcept { - if (data_.non_atomic_is_unique()) [[likely]] { - data_.non_atomic_reallocate(new_size); - } else { - SharedPtr new_data(new_size, constructed_item_count()); - PRAGMA_DIAGNOSTIC(push) - PRAGMA_DIAGNOSTIC(ignored DIAGNOSTIC_CLASS_MEMACCESS) - std::memcpy(new_data.get(), data_.get(), size_ * sizeof(T)); - PRAGMA_DIAGNOSTIC(pop) - swap(data_, new_data); - } - + data_.reallocate(size_, new_size); size_ = new_size; } diff --git a/pp/bare_bones/stream_v_byte.h b/pp/bare_bones/stream_v_byte.h index ee058de261..54eaa9cd79 100644 --- a/pp/bare_bones/stream_v_byte.h +++ b/pp/bare_bones/stream_v_byte.h @@ -960,7 +960,7 @@ class CompactSequence { [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t size() const noexcept { if constexpr (IsSharedMemory>::value) { - return buffer_.constructed_item_count(); + return buffer_.items_count(); } else if constexpr (kIsReadOnly) { return buffer_.size(); } else { @@ -1016,7 +1016,7 @@ class CompactSequence { private: PROMPP_ALWAYS_INLINE void set_size(uint32_t new_size) noexcept { if constexpr (IsSharedMemory>::value) { - buffer_.set_constructed_item_count(new_size); + buffer_.set_items_count(new_size); } else { buffer_.control_block().items_count = new_size; } diff --git a/pp/bare_bones/tests/bit_sequence_tests.cpp b/pp/bare_bones/tests/bit_sequence_tests.cpp index 83bf181cbc..5da1ffc0bc 100644 --- a/pp/bare_bones/tests/bit_sequence_tests.cpp +++ b/pp/bare_bones/tests/bit_sequence_tests.cpp @@ -13,7 +13,7 @@ using BareBones::CompactBitSequence; constexpr size_t NUM_VALUES = 1000; -constexpr std::array kAllocationSizesTable = {AllocationSize{0U}, AllocationSize{32U}}; +constexpr std::array kAllocationSizesTable = {AllocationSize{0U}, AllocationSize{32U}, AllocationSize{64U}}; std::array generate_uint64_vector() { std::array data; @@ -196,6 +196,56 @@ class CompactBitSequenceFixture : public testing::Test { CompactBitSequence stream_; }; +TEST_F(CompactBitSequenceFixture, CopyConstructor) { + // Arrange + stream_.push_back_single_one_bit(); + + // Act + const auto stream2 = stream_; + + // Assert + EXPECT_EQ(1U, stream_.size_in_bits()); + EXPECT_EQ(1U, stream2.size_in_bits()); + EXPECT_NE(stream_.raw_bytes(), stream2.raw_bytes()); + EXPECT_EQ(0b1U, stream_.bytes()[0]); + EXPECT_EQ(0b1U, stream2.bytes()[0]); +} + +TEST_F(CompactBitSequenceFixture, CopyOperator) { + // Arrange + stream_.push_back_single_one_bit(); + decltype(stream_) stream2; + stream2.push_back_single_zero_bit(); + + // Act + stream2 = stream_; + + // Assert + EXPECT_EQ(1U, stream_.size_in_bits()); + EXPECT_EQ(1U, stream2.size_in_bits()); + EXPECT_NE(stream_.raw_bytes(), stream2.raw_bytes()); + EXPECT_EQ(0b1U, stream2.bytes()[0]); +} + +TEST_F(CompactBitSequenceFixture, CopyOperatorOnNonUniqueMemory) { + // Arrange + stream_.push_back_single_one_bit(); + decltype(stream_) stream2; + stream2.push_back_single_zero_bit(); + const auto memory = stream2.shared_memory(); + + // Act + stream2 = stream_; + + // Assert + EXPECT_EQ(1U, stream_.size_in_bits()); + EXPECT_EQ(1U, stream2.size_in_bits()); + EXPECT_NE(stream_.raw_bytes(), stream2.raw_bytes()); + EXPECT_NE(stream2.raw_bytes(), memory.get()); + EXPECT_EQ(0b1U, stream2.bytes()[0]); + EXPECT_EQ(0b0U, memory.get()[0]); +} + TEST_F(CompactBitSequenceFixture, MoveConstructor) { // Arrange stream_.push_back_single_one_bit(); @@ -230,6 +280,26 @@ TEST_F(CompactBitSequenceFixture, MoveOperator) { EXPECT_EQ(0b1U, stream2.bytes()[0]); } +TEST_F(CompactBitSequenceFixture, MoveOperatorOnNonUniqueMemory) { + // Arrange + stream_.push_back_single_one_bit(); + const auto memory = stream_.shared_memory(); + decltype(stream_) stream2; + stream2.push_back_single_zero_bit(); + const auto memory2 = stream2.shared_memory(); + + // Act + stream2 = std::move(stream_); + + // Assert + EXPECT_EQ(1U, stream2.size_in_bits()); + ASSERT_FALSE(stream2.bytes().empty()); + EXPECT_EQ(0b1U, stream2.bytes()[0]); + + EXPECT_EQ(stream2.raw_bytes(), memory.get()); + EXPECT_EQ(0b0U, memory2.get()[0]); +} + TEST_F(CompactBitSequenceFixture, PushOnebit) { // Arrange @@ -337,7 +407,7 @@ TEST_F(CompactBitSequenceFixture, TrimUint32) { // Act stream_.push_back_bits_u32(32, 0b10101010101010101010101010101010); stream_.trim_lower_bytes(2); - auto bytes = stream_.bytes().data(); + const auto bytes = stream_.bytes().data(); // Assert ASSERT_EQ(16U, stream_.size_in_bits()); @@ -352,7 +422,7 @@ TEST_F(CompactBitSequenceFixture, TrimUint32_2) { stream_.push_back_bits_u32(32, 0b10101010101010101010101010101010); stream_.trim_lower_bytes(4); - auto bytes = stream_.bytes().data(); + const auto bytes = stream_.bytes().data(); // Assert ASSERT_EQ(1U, stream_.size_in_bits()); @@ -367,7 +437,7 @@ TEST_F(CompactBitSequenceFixture, TrimUint32_3) { stream_.push_back_bits_u32(32, 0b10101010101010101010101010101010); stream_.trim_lower_bytes(3); - auto bytes = stream_.bytes().data(); + const auto bytes = stream_.bytes().data(); // Assert ASSERT_EQ(9U, stream_.size_in_bits()); @@ -380,7 +450,7 @@ TEST_F(CompactBitSequenceFixture, TrimUint64) { // Act stream_.push_back_u64(0b1010101010101010101010101010101010101010101010101010101010101010); stream_.trim_lower_bytes(5); - auto bytes = stream_.bytes().data(); + const auto bytes = stream_.bytes().data(); // Assert ASSERT_EQ(24U, stream_.size_in_bits()); @@ -394,13 +464,69 @@ TEST_F(CompactBitSequenceFixture, TrimUint64_2) { stream_.push_back_single_zero_bit(); stream_.push_back_u64(0b1010101010101010101010101010101010101010101010101010101010101010); stream_.trim_lower_bytes(8); - auto bytes = stream_.bytes().data(); + const auto bytes = stream_.bytes().data(); // Assert ASSERT_EQ(1U, stream_.size_in_bits()); EXPECT_EQ(0b1ULL, bytes[0]); } +TEST_F(CompactBitSequenceFixture, ShrinkToFit) { + // Arrange + static constexpr auto kValue = std::numeric_limits::max(); + + stream_.push_back_u64(kValue); + const auto allocated_memory = stream_.allocated_memory(); + + // Act + stream_.shrink_to_fit(); + + // Assert + EXPECT_LT(stream_.allocated_memory(), allocated_memory); + ASSERT_EQ(sizeof(kValue), stream_.size_in_bytes()); + EXPECT_EQ(kValue, *stream_.bytes().data()); +} + +TEST_F(CompactBitSequenceFixture, ShrinkToFitOnNonUniqueMemory) { + // Arrange + static constexpr auto kValue = std::numeric_limits::max(); + + stream_.push_back_u64(kValue); + const auto memory = stream_.shared_memory(); + const auto memory_size = stream_.size_in_bits(); + const auto allocated_memory = stream_.allocated_memory(); + + // Act + stream_.shrink_to_fit(); + + // Assert + EXPECT_LT(stream_.allocated_memory(), allocated_memory); + ASSERT_EQ(sizeof(kValue), stream_.size_in_bytes()); + EXPECT_EQ(kValue, *stream_.bytes().data()); + ASSERT_EQ(BareBones::Bit::to_bits(sizeof(kValue)), memory_size); + EXPECT_EQ(kValue, *reinterpret_cast(memory.get())); +} + +TEST_F(CompactBitSequenceFixture, ReallocOnNonUniqueMemory) { + // Arrange + static constexpr auto kValue = std::numeric_limits::max(); + + stream_.push_back_u64(kValue); + stream_.push_back_u64(kValue); + stream_.push_back_u64(kValue); + const auto memory = stream_.shared_memory(); + const auto memory_size = stream_.size_in_bits(); + + // Act + stream_.push_back_u64(kValue); + + // Assert + ASSERT_EQ(BareBones::Bit::to_bits(sizeof(kValue) * 3), memory_size); + EXPECT_NE(stream_.raw_bytes(), memory.get()); + EXPECT_TRUE(std::ranges::equal(std::vector{kValue, kValue, kValue}, std::span(reinterpret_cast(memory.get()), 3))); + EXPECT_TRUE(std::ranges::equal(std::vector{kValue, kValue, kValue, kValue}, stream_.bytes())); +} + template class BitSequenceReaderFixture : public testing::Test {}; diff --git a/pp/bare_bones/tests/memory_tests.cpp b/pp/bare_bones/tests/memory_tests.cpp index 7a08654ebb..4cb463b3f9 100644 --- a/pp/bare_bones/tests/memory_tests.cpp +++ b/pp/bare_bones/tests/memory_tests.cpp @@ -13,6 +13,7 @@ using BareBones::MemoryControlBlock; using BareBones::MemoryControlBlockWithItemCount; using BareBones::SharedMemory; using BareBones::SharedPtr; +using BareBones::SharedPtrControlBlockWithItemCount; struct GetAllocationSizeCase { uint32_t needed_size; @@ -181,7 +182,7 @@ TEST_F(MemoryWithItemCountFixture, CopyOperator) { class SharedPtrFixture : public ::testing::Test { protected: template - using SharedPtr = BareBones::SharedPtr; + using SharedPtr = BareBones::SharedPtr; }; TEST_F(SharedPtrFixture, Empty) { @@ -200,7 +201,7 @@ TEST_F(SharedPtrFixture, ConstructorWithSize) { // Arrange // Act - const SharedPtr ptr(16); + const SharedPtr ptr(16, 0); // Assert EXPECT_TRUE(ptr.non_atomic_is_unique()); @@ -208,12 +209,12 @@ TEST_F(SharedPtrFixture, ConstructorWithSize) { EXPECT_NE(nullptr, ptr.get()); } -TEST_F(SharedPtrFixture, NonAtomicReallocate) { +TEST_F(SharedPtrFixture, Reallocate) { // Arrange - SharedPtr ptr(16); + SharedPtr ptr(16, 0); // Act - ptr.non_atomic_reallocate(32); + ptr.reallocate(0, 32); ptr.get()[31] = '\x00'; // Assert @@ -223,7 +224,7 @@ TEST_F(SharedPtrFixture, NonAtomicReallocate) { TEST_F(SharedPtrFixture, CopyConstructor) { // Arrange - const SharedPtr ptr(16); + const SharedPtr ptr(16, 0); // Act // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) @@ -238,10 +239,10 @@ TEST_F(SharedPtrFixture, CopyConstructor) { TEST_F(SharedPtrFixture, CopyOperator) { // Arrange - const SharedPtr ptr(16); + const SharedPtr ptr(16, 0); // Act - SharedPtr ptr2(16); + SharedPtr ptr2(16, 0); ptr2 = ptr; // Assert @@ -253,7 +254,7 @@ TEST_F(SharedPtrFixture, CopyOperator) { TEST_F(SharedPtrFixture, MoveConstructor) { // Arrange - SharedPtr ptr(16); + SharedPtr ptr(16, 0); // Act const auto ptr2 = std::move(ptr); @@ -265,11 +266,11 @@ TEST_F(SharedPtrFixture, MoveConstructor) { TEST_F(SharedPtrFixture, MoveOperator) { // Arrange - SharedPtr ptr(16); + SharedPtr ptr(16, 0); const auto ptr_memory = ptr.get(); // Act - SharedPtr ptr2(16); + SharedPtr ptr2(16, 0); ptr2 = std::move(ptr); // Assert @@ -277,31 +278,43 @@ TEST_F(SharedPtrFixture, MoveOperator) { EXPECT_EQ(ptr_memory, ptr2.get()); } -TEST_F(SharedPtrFixture, DestructItems) { +TEST_F(SharedPtrFixture, ReallocateToLagerSize) { // Arrange - SharedPtr ptr(1); + SharedPtr ptr(1, 0); + std::construct_at(ptr.get(), "123456"); + ptr.set_items_count(1); // Act - const auto string = std::construct_at(ptr.get()); - ptr.set_constructed_item_count(1); - string->resize(32); + ptr.reallocate(1, 2); // Assert + EXPECT_EQ(1U, ptr.items_count()); + EXPECT_EQ("123456", ptr.get()[0]); } -TEST_F(SharedPtrFixture, ReallocateAtExistingMemory) { +TEST_F(SharedPtrFixture, ReallocateToZeroSize) { // Arrange - SharedPtr ptr(1); + SharedPtr ptr(1, 0); + std::construct_at(ptr.get()); // Act - const auto string = std::construct_at(ptr.get()); - ptr.set_constructed_item_count(1); - string->resize(32); + ptr.reallocate(1, 0); - ptr.non_atomic_reallocate(2); + // Assert + EXPECT_TRUE(ptr.non_atomic_is_unique()); +} + +TEST_F(SharedPtrFixture, ReallocateToSmallerSize) { + // Arrange + SharedPtr ptr(2, 0); + std::construct_at(&ptr.get()[0], "123456"); + std::construct_at(&ptr.get()[1], "654321"); + + // Act + ptr.reallocate(2, 1); // Assert - EXPECT_EQ(1U, ptr.constructed_item_count()); + EXPECT_EQ("123456", ptr.get()[0]); } class SharedMemoryFixture : public ::testing::Test { @@ -382,7 +395,7 @@ TEST_F(SharedMemoryFixture, MoveOperator) { TEST_F(SharedMemoryFixture, ResizeOnNonUniqueOwner) { // Arrange memory_.resize_to_fit_at_least(1); - memory_.set_constructed_item_count(1); + memory_.set_items_count(1); auto memory2 = memory_; // Act @@ -391,7 +404,7 @@ TEST_F(SharedMemoryFixture, ResizeOnNonUniqueOwner) { // Assert EXPECT_NE(memory_.size(), memory2.size()); EXPECT_NE(memory_.begin(), memory2.begin()); - EXPECT_EQ(1U, memory2.constructed_item_count()); + EXPECT_EQ(1U, memory2.items_count()); } } // namespace \ No newline at end of file diff --git a/pp/bare_bones/tests/vector_tests.cpp b/pp/bare_bones/tests/vector_tests.cpp index 2bf50f0983..b3e5124e8d 100644 --- a/pp/bare_bones/tests/vector_tests.cpp +++ b/pp/bare_bones/tests/vector_tests.cpp @@ -85,7 +85,7 @@ TEST(BareBonesVector, InitializerListConstructor) { // Arrange // Act - Vector vector{"123", "456", "789"}; + Vector vector{"123", "456", "789"}; // Assert EXPECT_EQ(3U, vector.size()); diff --git a/pp/bare_bones/type_traits.h b/pp/bare_bones/type_traits.h index 8c0db70901..2be0531899 100644 --- a/pp/bare_bones/type_traits.h +++ b/pp/bare_bones/type_traits.h @@ -25,9 +25,6 @@ concept SubtractSemigroup = requires(T t1, T t2, T t3) { template struct IsTriviallyReallocatable : std::is_trivially_copyable {}; -template <> -struct IsTriviallyReallocatable : std::true_type {}; - template struct IsTriviallyReallocatable> : std::conjunction, IsTriviallyReallocatable> {}; diff --git a/pp/bare_bones/vector.h b/pp/bare_bones/vector.h index cd58598b96..c639030f61 100644 --- a/pp/bare_bones/vector.h +++ b/pp/bare_bones/vector.h @@ -387,8 +387,8 @@ class SharedVector : public GenericVector, typename [[nodiscard]] PROMPP_ALWAYS_INLINE auto& memory() noexcept { return memory_; } [[nodiscard]] PROMPP_ALWAYS_INLINE const auto& memory() const noexcept { return memory_; } - [[nodiscard]] PROMPP_ALWAYS_INLINE SizeType get_size() const noexcept { return memory_.constructed_item_count(); } - PROMPP_ALWAYS_INLINE void set_size(SizeType size) noexcept { memory_.set_constructed_item_count(size); } + [[nodiscard]] PROMPP_ALWAYS_INLINE SizeType get_size() const noexcept { return memory_.items_count(); } + PROMPP_ALWAYS_INLINE void set_size(SizeType size) noexcept { memory_.set_items_count(size); } private: SharedMemory memory_; @@ -419,11 +419,13 @@ class SharedSpan { template requires std::is_trivially_destructible_v - explicit SharedSpan(const SharedVector& vector) : data_(reinterpret_cast&>(vector.shared_ptr())) {} + explicit SharedSpan(const SharedVector& vector) + : data_(reinterpret_cast&>(vector.shared_ptr())) {} template requires std::is_trivially_destructible_v - explicit SharedSpan(const SharedMemory& memory) : data_(reinterpret_cast&>(memory.ptr())) {} + explicit SharedSpan(const SharedMemory& memory) + : data_(reinterpret_cast&>(memory.ptr())) {} SharedSpan(const SharedSpan&) = default; SharedSpan(SharedSpan&& other) noexcept : data_(std::move(other.data_)) {} @@ -446,13 +448,13 @@ class SharedSpan { return data_.get()[i]; } - [[nodiscard]] PROMPP_ALWAYS_INLINE SizeType size() const noexcept { return data_.constructed_item_count(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE SizeType size() const noexcept { return data_.items_count(); } [[nodiscard]] PROMPP_ALWAYS_INLINE const T* data() const noexcept { return begin(); } [[nodiscard]] PROMPP_ALWAYS_INLINE const T* begin() const noexcept { return data_.get(); } [[nodiscard]] PROMPP_ALWAYS_INLINE const T* end() const noexcept { return begin() + size(); } private: - SharedPtr data_; + SharedPtr data_; }; template diff --git a/pp/entrypoint/series_data/querier.h b/pp/entrypoint/series_data/querier.h index f67ca505e4..4245988e06 100644 --- a/pp/entrypoint/series_data/querier.h +++ b/pp/entrypoint/series_data/querier.h @@ -6,7 +6,6 @@ #include "series_data/querier/instant_querier.h" #include "series_data/querier/querier.h" #include "series_data/serialization/serialized_data.h" -#include "series_data/serialization/serializer.h" namespace entrypoint::series_data { @@ -50,51 +49,12 @@ class InstantQuerierWithArgumentsWrapper { using InstantQuerierWithArgumentsWrapperEntrypoint = InstantQuerierWithArgumentsWrapper, PromPP::Primitives::Go::SliceView<::series_data::encoder::Sample>>; -class RangeQuerierWithArgumentsWrapper { - using DataStorage = ::series_data::DataStorage; - using LabelSetID = PromPP::Primitives::LabelSetID; - template - using Slice = PromPP::Primitives::Go::Slice; - using Query = ::series_data::querier::Query>; - using Serializer = ::series_data::serialization::Serializer; - using BytesStream = PromPP::Primitives::Go::BytesStream; - - public: - RangeQuerierWithArgumentsWrapper(DataStorage& storage, const Query& query, Slice* serialized_chunks) - : querier_(storage), query_(&query), serialized_chunks_(serialized_chunks) {} - - void query() noexcept { - querier_.query(*query_); - if (!querier_.need_loading()) { - serialize_chunks(); - } - } - - PROMPP_ALWAYS_INLINE void query_finalize() const noexcept { serialize_chunks(); } - - [[nodiscard]] const BareBones::Bitset& series_to_load() const noexcept { return querier_.get_series_to_load(); } - [[nodiscard]] bool need_loading() const noexcept { return querier_.need_loading(); } - [[nodiscard]] DataStorage& storage() noexcept { return querier_.get_storage(); } - - private: - ::series_data::querier::Querier querier_; - const Query* query_; - Slice* serialized_chunks_; - - PROMPP_ALWAYS_INLINE void serialize_chunks() const noexcept { - Serializer serializer{querier_.get_storage()}; - PromPP::Primitives::Go::BytesStream bytes_stream{serialized_chunks_}; - serializer.serialize(querier_.chunks(), bytes_stream); - } -}; - class RangeQuerierWithArgumentsWrapperV2 { using DataStorage = ::series_data::DataStorage; using LabelSetID = PromPP::Primitives::LabelSetID; template using Slice = PromPP::Primitives::Go::Slice; using Query = ::series_data::querier::Query>; - using Serializer = ::series_data::serialization::Serializer; using BytesStream = PromPP::Primitives::Go::BytesStream; public: @@ -126,11 +86,10 @@ class RangeQuerierWithArgumentsWrapperV2 { enum class QuerierType : uint8_t { kInstantQuerier = 0, kRangeQuerier, kRangeQuerierV2 }; -using QuerierVariant = std::variant; +using QuerierVariant = std::variant; using QuerierVariantPtr = std::unique_ptr; } // namespace entrypoint::series_data static_assert(entrypoint::series_data::QuerierInterface); -static_assert(entrypoint::series_data::QuerierInterface); static_assert(entrypoint::series_data::QuerierInterface); \ No newline at end of file diff --git a/pp/entrypoint/series_data_data_storage.cpp b/pp/entrypoint/series_data_data_storage.cpp index 0cb19dd774..adc36224d4 100644 --- a/pp/entrypoint/series_data_data_storage.cpp +++ b/pp/entrypoint/series_data_data_storage.cpp @@ -12,7 +12,6 @@ #include "series_data/querier.h" #include "series_data/querier/instant_querier.h" #include "series_data/querier/querier.h" -#include "series_data/serialization/serializer.h" #include "series_data/unloading/loader.h" #include "series_data/unloading/unloader.h" #include "series_index/querier/selector_querier.h" @@ -112,37 +111,6 @@ extern "C" void prompp_series_data_data_storage_queried_series_set_bitset(void* new (res) Result{.result = result}; } -extern "C" void prompp_series_data_data_storage_query(void* args, void* res) { - using Query = series_data::querier::Query>; - using entrypoint::series_data::RangeQuerierWithArgumentsWrapper; - using series_data::querier::Querier; - - struct Arguments { - DataStoragePtr data_storage; - Query query; - Slice* serialized_chunks; - }; - - struct Result { - QuerierVariantPtr querier{}; - QueryStatus status; - }; - - const auto in = static_cast(args); - - RangeQuerierWithArgumentsWrapper querier(*in->data_storage, in->query, in->serialized_chunks); - querier.query(); - - if (querier.need_loading()) { - new (res) Result{ - .querier = std::make_unique(std::in_place_index<1>, std::move(querier)), - .status = QueryStatus::kNeedDataLoad, - }; - } else { - new (res) Result{.status = QueryStatus::kSuccess}; - } -} - extern "C" void prompp_series_data_data_storage_query_v2(void* args, void* res) { using Query = series_data::querier::Query>; using entrypoint::series_data::RangeQuerierWithArgumentsWrapperV2; @@ -166,7 +134,7 @@ extern "C" void prompp_series_data_data_storage_query_v2(void* args, void* res) querier.query(); if (querier.need_loading()) { - out->querier = std::make_unique(std::in_place_index<2>, std::move(querier)); + out->querier = std::make_unique(std::in_place_index<1>, std::move(querier)); out->status = QueryStatus::kNeedDataLoad; } else { out->status = QueryStatus::kSuccess; @@ -296,6 +264,7 @@ extern "C" void prompp_series_data_chunk_recoder_recode_next_chunk(void* args, v const auto in = static_cast(args); const auto out = static_cast(res); + // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) std::visit( [out](auto& chunk_recoder) PROMPP_LAMBDA_INLINE { chunk_recoder.recode_next_chunk(*out); diff --git a/pp/entrypoint/series_data_data_storage.h b/pp/entrypoint/series_data_data_storage.h index fcc4474dea..ed8412de4c 100644 --- a/pp/entrypoint/series_data_data_storage.h +++ b/pp/entrypoint/series_data_data_storage.h @@ -92,22 +92,6 @@ void prompp_series_data_data_storage_queried_series_set_bitset(void* args, void* */ void prompp_series_data_data_storage_allocated_memory(void* args, void* res); -/** - * @brief Queries data storage and serializes result. - * - * @param args { - * dataStorage uintptr // pointer to constructed data storage - * query DataStorageQuery // query - * serializedData *[]byte // pointer to slice for serialized data - * } - * - * @param res { - * Querier uintptr // pointer to constructed Querier if data loading is needed - * Status uint8 // status of a query (0 - Success, 1 - Data loading is needed) - * } - */ -void prompp_series_data_data_storage_query(void* args, void* res); - /** * @brief Queries data storage and serializes result (new serialization model). * diff --git a/pp/go/cppbridge/entrypoint.h b/pp/go/cppbridge/entrypoint.h index 2d1b7061ae..d2bcb88ec7 100755 --- a/pp/go/cppbridge/entrypoint.h +++ b/pp/go/cppbridge/entrypoint.h @@ -1327,22 +1327,6 @@ void prompp_series_data_data_storage_queried_series_set_bitset(void* args, void* */ void prompp_series_data_data_storage_allocated_memory(void* args, void* res); -/** - * @brief Queries data storage and serializes result. - * - * @param args { - * dataStorage uintptr // pointer to constructed data storage - * query DataStorageQuery // query - * serializedData *[]byte // pointer to slice for serialized data - * } - * - * @param res { - * Querier uintptr // pointer to constructed Querier if data loading is needed - * Status uint8 // status of a query (0 - Success, 1 - Data loading is needed) - * } - */ -void prompp_series_data_data_storage_query(void* args, void* res); - /** * @brief Queries data storage and serializes result (new serialization model). * diff --git a/pp/head/chunk_recoder_tests.cpp b/pp/head/chunk_recoder_tests.cpp index 2455fb5d84..0fb1bb2ea9 100644 --- a/pp/head/chunk_recoder_tests.cpp +++ b/pp/head/chunk_recoder_tests.cpp @@ -3,7 +3,7 @@ #include "chunk_recoder.h" #include "series_data/encoder.h" #include "series_data/serialization/deserializer.h" -#include "series_data/serialization/serializer.h" +#include "series_data/serialization/serialized_data.h" namespace { @@ -14,7 +14,7 @@ using PromPP::Primitives::LabelSetID; using PromPP::Primitives::TimeInterval; using series_data::DataStorage; using series_data::Encoder; -using series_data::serialization::Serializer; +using series_data::serialization::DataSerializer; using std::operator""s; class ChunkRecoderFixture : public ::testing::Test { @@ -407,11 +407,9 @@ TEST_F(ChunkRecoderFixture, RecodeSerializedChunks) { encoder.encode(4, 3, 2.0); encoder.encode(4, 4, 2.0); - Serializer serializer{storage_}; - BareBones::ShrinkedToFitOStringStream stream; - serializer.serialize(stream); - - ChunkRecoder recoder(series_data::chunk::SerializedChunkIterator{stream.span()}, {.min = 0, .max = 4}); + DataSerializer serializer{storage_}; + const auto serialized_data = serializer.serialize(); + ChunkRecoder recoder(series_data::chunk::SerializedChunkIterator{serialized_data.bytes_buffer, serialized_data.chunks}, {.min = 0, .max = 4}); // Act const auto info1 = recode(recoder); @@ -449,6 +447,7 @@ TEST_F(ChunkRecoderFixture, RecodeWithLsIdBatchSize) { auto recoder = create_recoder({0, 1, 2, 3}, 1, {.min = 0, .max = 4}); // Act + // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) const auto info1 = recode(recoder); const auto next_batch_result1 = recoder.chunk_iterator().next_batch(); const auto next_batch_result2 = recoder.chunk_iterator().next_batch(); diff --git a/pp/primitives/snug_composites_filaments.h b/pp/primitives/snug_composites_filaments.h index b2f7421f63..bf276e9fe9 100644 --- a/pp/primitives/snug_composites_filaments.h +++ b/pp/primitives/snug_composites_filaments.h @@ -463,7 +463,8 @@ class LabelSet { static constexpr bool kIsReadOnly = BareBones::IsSharedSpan>::value; public: - using symbols_tables_type = std::conditional_t>, Vector>>>; + using symbols_tables_type = + std::conditional_t>, BareBones::Vector>>>; using symbols_ids_sequences_type = Vector; diff --git a/pp/prometheus/tsdb/chunkenc/bstream.h b/pp/prometheus/tsdb/chunkenc/bstream.h index ca7bafb559..f455f554db 100644 --- a/pp/prometheus/tsdb/chunkenc/bstream.h +++ b/pp/prometheus/tsdb/chunkenc/bstream.h @@ -53,6 +53,7 @@ class BStream : public BareBones::CompactBitSequenceBase #include "bare_bones/preprocess.h" -#include "primitives/go_slice.h" #include "profiling/profiling.h" #include "series_data/encoder.h" #include "series_data/querier/query.h" #include "series_data/serialization/serialized_data.h" -#include "series_data/serialization/serializer.h" namespace { @@ -64,84 +62,6 @@ series_data::querier::QueriedChunkList generate_query(uint32_t size) { return chunk_list; } -void BenchmarkWalSerializer(benchmark::State& state) { - ZoneScoped; - const auto& samples = get_samples_for_benchmark(); - const double percent = static_cast(state.range(0)) / 100.0; - const auto [min, max] = std::ranges::minmax_element(samples, [](auto a, auto b) { return a.timestamp < b.timestamp; }); - const auto min_ts = min->timestamp; - const auto max_ts = max->timestamp; - const auto delta_ts = max_ts - min_ts; - - series_data::DataStorage storage; - series_data::Encoder encoder{storage}; - - for (const auto& sample : samples) { - if (sample.timestamp <= min_ts + static_cast(static_cast(delta_ts) * percent)) { - encoder.encode(sample.series_id, sample.timestamp, sample.value); - } - } - - const series_data::querier::QueriedChunkList chunk_list = generate_query(storage.open_chunks.size()); - - for ([[maybe_unused]] auto _ : state) { - series_data::serialization::Serializer serializer_{storage}; - PromPP::Primitives::Go::Slice slice; - PromPP::Primitives::Go::BytesStream stream{&slice}; - - serializer_.serialize(chunk_list, stream); - } - - { - series_data::serialization::Serializer serializer_{storage}; - PromPP::Primitives::Go::Slice slice; - PromPP::Primitives::Go::BytesStream stream{&slice}; - - serializer_.serialize(chunk_list, stream); - state.counters["Stream Size"] = - benchmark::Counter(static_cast(slice.allocated_memory()), benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024); - } -} - -void BenchmarkWalConstantSerializer(benchmark::State& state) { - ZoneScoped; - const auto& samples = get_samples_for_benchmark(); - const double percent = static_cast(state.range(0)) / 100.0; - const auto [min, max] = std::ranges::minmax_element(samples, [](auto a, auto b) { return a.timestamp < b.timestamp; }); - const auto min_ts = min->timestamp; - const auto max_ts = max->timestamp; - const auto delta_ts = max_ts - min_ts; - - series_data::DataStorage storage; - series_data::Encoder encoder{storage}; - - for (const auto& sample : samples) { - if (sample.timestamp <= min_ts + static_cast(static_cast(delta_ts) * percent)) { - encoder.encode(sample.series_id, sample.timestamp, sample.series_id); - } - } - - const series_data::querier::QueriedChunkList chunk_list = generate_query(storage.open_chunks.size()); - - for ([[maybe_unused]] auto _ : state) { - series_data::serialization::Serializer serializer_{storage}; - PromPP::Primitives::Go::Slice slice; - PromPP::Primitives::Go::BytesStream stream{&slice}; - - serializer_.serialize(chunk_list, stream); - } - - { - series_data::serialization::Serializer serializer_{storage}; - PromPP::Primitives::Go::Slice slice; - PromPP::Primitives::Go::BytesStream stream{&slice}; - - serializer_.serialize(chunk_list, stream); - state.counters["Stream Size"] = - benchmark::Counter(static_cast(slice.allocated_memory()), benchmark::Counter::kDefaults, benchmark::Counter::OneK::kIs1024); - } -} - void BenchmarkWalSerializedData(benchmark::State& state) { ZoneScoped; const auto& samples = get_samples_for_benchmark(); @@ -204,9 +124,7 @@ void BenchmarkWalConstantSerializedData(benchmark::State& state) { } } -BENCHMARK(BenchmarkWalSerializer)->Arg(25)->Arg(50)->Arg(75)->Arg(100); BENCHMARK(BenchmarkWalSerializedData)->Arg(25)->Arg(50)->Arg(75)->Arg(100); -BENCHMARK(BenchmarkWalConstantSerializer)->Arg(25)->Arg(50)->Arg(75)->Arg(100); BENCHMARK(BenchmarkWalConstantSerializedData)->Arg(25)->Arg(50)->Arg(75)->Arg(100); } // namespace diff --git a/pp/series_data/decoder.h b/pp/series_data/decoder.h index ede273f8ce..f0c1315cfc 100644 --- a/pp/series_data/decoder.h +++ b/pp/series_data/decoder.h @@ -14,6 +14,23 @@ #include "primitives/primitives.h" namespace series_data { + +#pragma pack(push, 1) + +struct SerializedCompactBitSequence { + template + PROMPP_ALWAYS_INLINE explicit SerializedCompactBitSequence(const CompactBitSequence& bit_sequence) + : ptr(bit_sequence.shared_memory()), size_in_bits(bit_sequence.size_in_bits()) {} + + [[nodiscard]] PROMPP_ALWAYS_INLINE std::span buffer() const noexcept { return {ptr.get(), BareBones::Bit::to_ceil_bytes(size_in_bits)}; } + [[nodiscard]] PROMPP_ALWAYS_INLINE BareBones::BitSequenceReader reader() const noexcept { return {ptr.get(), size_in_bits}; } + + encoder::CompactBitSequence::SharedPtr ptr; + uint32_t size_in_bits; +}; + +#pragma pack(pop) + class Decoder { public: template @@ -184,83 +201,82 @@ class Decoder { switch (chunk.encoding_state.encoding_type) { case kUint32Constant: { - const auto timestamp_buffer = buffer.subspan(chunk.timestamps_offset); - std::forward(callback)( - decoder::ConstantDecodeIterator(BitSequenceWithItemsCount::count(timestamp_buffer.data()), BitSequenceWithItemsCount::reader(timestamp_buffer), - chunk.values_offset, chunk.encoding_state.has_last_stalenan), - DecodeIteratorSentinel{}); + const auto timestamp_bit_sequence = reinterpret_cast(buffer.data() + chunk.timestamps_offset); + std::forward(callback)(decoder::ConstantDecodeIterator(BitSequenceWithItemsCount::count(timestamp_bit_sequence->ptr.get()), + BitSequenceWithItemsCount::reader(timestamp_bit_sequence->buffer()), + chunk.values_offset, chunk.encoding_state.has_last_stalenan), + DecodeIteratorSentinel{}); break; } case kFloat32Constant: { - const auto timestamp_buffer = buffer.subspan(chunk.timestamps_offset); - std::forward(callback)( - decoder::ConstantDecodeIterator(BitSequenceWithItemsCount::count(timestamp_buffer.data()), BitSequenceWithItemsCount::reader(timestamp_buffer), - std::bit_cast(chunk.values_offset), chunk.encoding_state.has_last_stalenan), - DecodeIteratorSentinel{}); + const auto timestamp_bit_sequence = reinterpret_cast(buffer.data() + chunk.timestamps_offset); + std::forward(callback)(decoder::ConstantDecodeIterator(BitSequenceWithItemsCount::count(timestamp_bit_sequence->ptr.get()), + BitSequenceWithItemsCount::reader(timestamp_bit_sequence->buffer()), + std::bit_cast(chunk.values_offset), chunk.encoding_state.has_last_stalenan), + DecodeIteratorSentinel{}); break; } case kDoubleConstant: { - const auto timestamp_buffer = buffer.subspan(chunk.timestamps_offset); + const auto timestamp_bit_sequence = reinterpret_cast(buffer.data() + chunk.timestamps_offset); const auto values_buffer = buffer.subspan(chunk.values_offset); assert(values_buffer.size() >= sizeof(double)); std::forward(callback)( - decoder::ConstantDecodeIterator(BitSequenceWithItemsCount::count(timestamp_buffer.data()), BitSequenceWithItemsCount::reader(timestamp_buffer), + decoder::ConstantDecodeIterator(BitSequenceWithItemsCount::count(timestamp_bit_sequence->ptr.get()), + BitSequenceWithItemsCount::reader(timestamp_bit_sequence->buffer()), *reinterpret_cast(values_buffer.data()), chunk.encoding_state.has_last_stalenan), DecodeIteratorSentinel{}); break; } case kTwoDoubleConstant: { - const auto timestamp_buffer = buffer.subspan(chunk.timestamps_offset); + const auto timestamp_bit_sequence = reinterpret_cast(buffer.data() + chunk.timestamps_offset); const auto values_buffer = buffer.subspan(chunk.values_offset); assert(values_buffer.size() >= sizeof(encoder::value::TwoDoubleConstantEncoder)); std::forward(callback)( decoder::TwoDoubleConstantDecodeIterator( - BitSequenceWithItemsCount::count(timestamp_buffer.data()), BitSequenceWithItemsCount::reader(timestamp_buffer), + BitSequenceWithItemsCount::count(timestamp_bit_sequence->ptr.get()), BitSequenceWithItemsCount::reader(timestamp_bit_sequence->buffer()), *reinterpret_cast(values_buffer.data()), chunk.encoding_state.has_last_stalenan), DecodeIteratorSentinel{}); break; } case kAscInteger: { - const auto timestamp_buffer = buffer.subspan(chunk.timestamps_offset); - const auto values_buffer = buffer.subspan(chunk.values_offset); - std::forward(callback)( - decoder::AscIntegerDecodeIterator(BitSequenceWithItemsCount::count(timestamp_buffer.data()), BitSequenceWithItemsCount::reader(timestamp_buffer), - BareBones::BitSequenceReader(values_buffer.data(), BareBones::Bit::to_bits(values_buffer.size())), - chunk.encoding_state.has_last_stalenan), - DecodeIteratorSentinel{}); + const auto timestamp_bit_sequence = reinterpret_cast(buffer.data() + chunk.timestamps_offset); + const auto values_bit_sequence = reinterpret_cast(buffer.data() + chunk.values_offset); + std::forward(callback)(decoder::AscIntegerDecodeIterator(BitSequenceWithItemsCount::count(timestamp_bit_sequence->ptr.get()), + BitSequenceWithItemsCount::reader(timestamp_bit_sequence->buffer()), + values_bit_sequence->reader(), chunk.encoding_state.has_last_stalenan), + DecodeIteratorSentinel{}); break; } case kAscIntegerThenValuesGorilla: { - const auto timestamp_buffer = buffer.subspan(chunk.timestamps_offset); - const auto values_buffer = buffer.subspan(chunk.values_offset); + const auto timestamp_bit_sequence = reinterpret_cast(buffer.data() + chunk.timestamps_offset); + const auto values_bit_sequence = reinterpret_cast(buffer.data() + chunk.values_offset); std::forward(callback)( - decoder::AscIntegerThenValuesGorillaDecodeIterator( - BitSequenceWithItemsCount::count(timestamp_buffer.data()), BitSequenceWithItemsCount::reader(timestamp_buffer), - BareBones::BitSequenceReader(values_buffer.data(), BareBones::Bit::to_bits(values_buffer.size())), chunk.encoding_state.has_last_stalenan), + decoder::AscIntegerThenValuesGorillaDecodeIterator(BitSequenceWithItemsCount::count(timestamp_bit_sequence->ptr.get()), + BitSequenceWithItemsCount::reader(timestamp_bit_sequence->buffer()), + values_bit_sequence->reader(), chunk.encoding_state.has_last_stalenan), DecodeIteratorSentinel{}); break; } case kValuesGorilla: { - const auto timestamp_buffer = buffer.subspan(chunk.timestamps_offset); - const auto values_buffer = buffer.subspan(chunk.values_offset); - std::forward(callback)( - decoder::ValuesGorillaDecodeIterator(BitSequenceWithItemsCount::count(timestamp_buffer.data()), BitSequenceWithItemsCount::reader(timestamp_buffer), - BareBones::BitSequenceReader(values_buffer.data(), BareBones::Bit::to_bits(values_buffer.size())), - chunk.encoding_state.has_last_stalenan), - DecodeIteratorSentinel{}); + const auto timestamp_bit_sequence = reinterpret_cast(buffer.data() + chunk.timestamps_offset); + const auto values_bit_sequence = reinterpret_cast(buffer.data() + chunk.values_offset); + std::forward(callback)(decoder::ValuesGorillaDecodeIterator(BitSequenceWithItemsCount::count(timestamp_bit_sequence->ptr.get()), + BitSequenceWithItemsCount::reader(timestamp_bit_sequence->buffer()), + values_bit_sequence->reader(), chunk.encoding_state.has_last_stalenan), + DecodeIteratorSentinel{}); break; } case kGorilla: { - const auto values_buffer = buffer.subspan(chunk.values_offset); + const auto bit_sequence = reinterpret_cast(buffer.data() + chunk.values_offset); std::forward(callback)( - decoder::GorillaDecodeIterator(BitSequenceWithItemsCount::count(values_buffer.data()), BitSequenceWithItemsCount::reader(values_buffer), + decoder::GorillaDecodeIterator(BitSequenceWithItemsCount::count(bit_sequence->ptr.get()), BitSequenceWithItemsCount::reader(bit_sequence->buffer()), chunk.encoding_state.has_last_stalenan), DecodeIteratorSentinel{}); break; @@ -436,4 +452,5 @@ class Decoder { } } }; + } // namespace series_data diff --git a/pp/series_data/serialization/serialized_data.h b/pp/series_data/serialization/serialized_data.h index 7d815f5b9d..9d1350e400 100644 --- a/pp/series_data/serialization/serialized_data.h +++ b/pp/series_data/serialization/serialized_data.h @@ -11,10 +11,59 @@ namespace series_data::serialization { struct SerializedData { using Memory = BareBones::Memory; + ~SerializedData() { + uint32_t timestamp_offset{kNoTimestampOffset}; + for (auto& chunk : chunks) { + destroy_chunk_data(chunk, timestamp_offset); + } + } + BareBones::Vector chunks; Memory bytes_buffer; [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t allocated_memory() const noexcept { return chunks.allocated_memory() + bytes_buffer.allocated_memory(); } + + private: + static constexpr uint32_t kNoTimestampOffset = std::numeric_limits::max(); + + PROMPP_ALWAYS_INLINE void destroy_chunk_data(const chunk::SerializedChunk& chunk, uint32_t& timestamp_offset) noexcept { + using enum EncodingType; + + switch (chunk.encoding_state.encoding_type) { + case kUint32Constant: + case kFloat32Constant: + case kDoubleConstant: + case kTwoDoubleConstant: { + destroy_timestamp_stream_if_needed(chunk, timestamp_offset); + break; + } + + case kAscInteger: + case kAscIntegerThenValuesGorilla: + case kValuesGorilla: { + destroy_timestamp_stream_if_needed(chunk, timestamp_offset); + std::destroy_at(reinterpret_cast(bytes_buffer + chunk.values_offset)); + break; + } + + case kGorilla: { + std::destroy_at(reinterpret_cast(bytes_buffer + chunk.values_offset)); + break; + } + + default: { + assert(chunk.encoding_state.encoding_type != kUnknown); + break; + }; + } + } + + PROMPP_ALWAYS_INLINE void destroy_timestamp_stream_if_needed(const chunk::SerializedChunk& chunk, uint32_t& timestamp_offset) { + if (timestamp_offset == kNoTimestampOffset || chunk.timestamps_offset > timestamp_offset) [[unlikely]] { + timestamp_offset = chunk.timestamps_offset; + std::destroy_at(reinterpret_cast(bytes_buffer + chunk.timestamps_offset)); + } + } }; class DataSerializer { @@ -192,11 +241,10 @@ class DataSerializer { template static void write_compact_bit_sequence(const CompactBitSequence& bit_sequence, SerializedData::Memory& buffer) noexcept { - const auto bytes_count = bit_sequence.size_in_bytes(); uint32_t& data_size = buffer.control_block().items_count; - buffer.grow_to_fit_at_least(data_size + bytes_count); - std::memcpy(buffer + data_size, bit_sequence.raw_bytes(), bytes_count); - data_size += bytes_count; + buffer.grow_to_fit_at_least(data_size + sizeof(SerializedCompactBitSequence)); + std::construct_at(reinterpret_cast(buffer + data_size), bit_sequence); + data_size += sizeof(SerializedCompactBitSequence); } const DataStorage& storage_; diff --git a/pp/series_data/serialization/serializer.h b/pp/series_data/serialization/serializer.h deleted file mode 100644 index c0a357bf5f..0000000000 --- a/pp/series_data/serialization/serializer.h +++ /dev/null @@ -1,311 +0,0 @@ -#pragma once - -#include "bare_bones/concepts.h" -#include "series_data/chunk/serialized_chunk.h" -#include "series_data/data_storage.h" -#include "series_data/encoder/bit_sequence.h" -#include "series_data/querier/query.h" - -namespace series_data::serialization { -class Serializer { - public: - explicit Serializer(const DataStorage& storage) : storage_(storage) {} - - template - void serialize(const querier::QueriedChunkList& queried_chunks, Stream& stream) { - serialize_impl(queried_chunks, stream); - } - - template - void serialize(Stream& stream) { - serialize_impl(storage_.chunks(), stream); - } - - private: - struct TimestampStreamsData { - using TimestampId = uint32_t; - using Offset = uint32_t; - - static constexpr Offset kInvalidOffset = std::numeric_limits::max(); - - phmap::flat_hash_map stream_offsets; - phmap::flat_hash_map finalized_stream_offsets; - }; - - using QueriedChunk = querier::QueriedChunk; - using QueriedChunkList = querier::QueriedChunkList; - using SerializedChunk = chunk::SerializedChunk; - using SerializedChunkList = chunk::SerializedChunkList; - - const DataStorage& storage_; - - template - void serialize_impl(const ChunkList& chunks, Stream& stream) { - const auto& kReservedBytesForReader = encoder::CompactBitSequence::reserved_bytes_for_reader(); - - TimestampStreamsData timestamp_streams_data; - { - uint32_t data_size = sizeof(uint32_t); - uint32_t chunk_count = get_chunk_count(chunks); - auto serialized_chunks = create_serialized_chunks(chunks, chunk_count, timestamp_streams_data, data_size); - - if constexpr (BareBones::concepts::has_reserve) { - stream.reserve(data_size + kReservedBytesForReader.size()); - } - - write_chunk_count(chunk_count, stream); - write_serialized_chunks(serialized_chunks, stream); - } - - write_chunks_data(chunks, timestamp_streams_data, stream); - stream.write(kReservedBytesForReader.data(), kReservedBytesForReader.size()); - } - - template - PROMPP_ALWAYS_INLINE static uint32_t get_chunk_count(const ChunkList& chunks) noexcept { - if constexpr (std::is_same_v) { - return chunks.non_empty_chunk_count(); - } else { - return chunks.size(); - } - } - - PROMPP_ALWAYS_INLINE static void write_chunk_count(uint32_t chunk_count, std::ostream& stream) { - stream.write(reinterpret_cast(&chunk_count), sizeof(chunk_count)); - } - - PROMPP_ALWAYS_INLINE static void write_serialized_chunks(const SerializedChunkList& chunks, std::ostream& stream) noexcept { - const auto size_in_bytes = chunks.size() * sizeof(SerializedChunk); - stream.write(reinterpret_cast(chunks.data()), static_cast(size_in_bytes)); - } - - template - [[nodiscard]] SerializedChunkList create_serialized_chunks(const ChunkList& chunks, - uint32_t chunk_count, - TimestampStreamsData& timestamp_streams_data, - uint32_t& data_size) const noexcept { - SerializedChunkList serialized_chunks; - serialized_chunks.reserve(chunk_count); - data_size += sizeof(SerializedChunk) * chunk_count; - - for (auto& chunk_data : chunks) { - using enum chunk::DataChunk::Type; - - if (chunk_data.is_open()) [[likely]] { - if (const auto& chunk = get_chunk(chunk_data); !chunk.is_empty()) [[likely]] { - fill_serialized_chunk(chunk, serialized_chunks.emplace_back(chunk_data.series_id()), timestamp_streams_data, data_size); - } - } else { - fill_serialized_chunk(get_chunk(chunk_data), serialized_chunks.emplace_back(chunk_data.series_id()), timestamp_streams_data, - data_size); - } - } - - return serialized_chunks; - } - - template - void fill_serialized_chunk(const chunk::DataChunk& chunk, - SerializedChunk& serialized_chunk, - TimestampStreamsData& timestamp_streams_data, - uint32_t& data_size) const noexcept { - using enum EncodingType; - - serialized_chunk.encoding_state = chunk.encoding_state; - - switch (chunk.encoding_state.encoding_type) { - case kUint32Constant: { - fill_timestamp_stream_offset(timestamp_streams_data, chunk.timestamp_encoder_state_id, serialized_chunk, data_size); - serialized_chunk.store_value_in_offset(chunk.encoder.uint32_constant); - break; - } - - case kFloat32Constant: { - fill_timestamp_stream_offset(timestamp_streams_data, chunk.timestamp_encoder_state_id, serialized_chunk, data_size); - serialized_chunk.store_value_in_offset(chunk.encoder.float32_constant); - break; - } - - case kDoubleConstant: { - fill_timestamp_stream_offset(timestamp_streams_data, chunk.timestamp_encoder_state_id, serialized_chunk, data_size); - serialized_chunk.set_offset(data_size); - data_size += sizeof(encoder::value::DoubleConstantEncoder); - break; - } - - case kTwoDoubleConstant: { - fill_timestamp_stream_offset(timestamp_streams_data, chunk.timestamp_encoder_state_id, serialized_chunk, data_size); - serialized_chunk.set_offset(data_size); - data_size += sizeof(encoder::value::TwoDoubleConstantEncoder); - break; - } - - case kAscInteger: { - fill_timestamp_stream_offset(timestamp_streams_data, chunk.timestamp_encoder_state_id, serialized_chunk, data_size); - serialized_chunk.set_offset(data_size); - data_size += storage_.get_asc_integer_stream(chunk.encoder.external_index).size_in_bytes(); - break; - } - - case kAscIntegerThenValuesGorilla: { - fill_timestamp_stream_offset(timestamp_streams_data, chunk.timestamp_encoder_state_id, serialized_chunk, data_size); - serialized_chunk.set_offset(data_size); - data_size += storage_.get_asc_integer_then_values_gorilla_stream(chunk.encoder.external_index).size_in_bytes(); - break; - } - - case kValuesGorilla: { - fill_timestamp_stream_offset(timestamp_streams_data, chunk.timestamp_encoder_state_id, serialized_chunk, data_size); - serialized_chunk.set_offset(data_size); - data_size += storage_.get_values_gorilla_stream(chunk.encoder.external_index).size_in_bytes(); - break; - } - - case kGorilla: { - serialized_chunk.set_offset(data_size); - data_size += storage_.get_gorilla_encoder_stream(chunk.encoder.external_index).size_in_bytes(); - break; - } - - default: { - assert(chunk.encoding_state.encoding_type != kUnknown); - } - } - } - - template - [[nodiscard]] const chunk::DataChunk& get_chunk(const QueriedChunk& queried_chunk) const noexcept { - if constexpr (chunk_type == chunk::DataChunk::Type::kOpen) { - return storage_.open_chunks[queried_chunk.series_id()]; - } else { - auto finalized_chunk_it = storage_.finalized_chunks.find(queried_chunk.series_id())->second.begin(); - std::advance(finalized_chunk_it, queried_chunk.finalized_chunk_id); - return *finalized_chunk_it; - } - } - - template - [[nodiscard]] static const chunk::DataChunk& get_chunk(const DataStorage::SeriesChunkIterator::Data& chunk) noexcept { - return chunk.chunk(); - } - - template - void fill_timestamp_stream_offset(TimestampStreamsData& timestamp_streams_data, - encoder::timestamp::State::Id timestamp_stream_id, - SerializedChunk& serialized_chunk, - uint32_t& data_size) const noexcept { - if constexpr (chunk_type == chunk::DataChunk::Type::kOpen) { - if (const auto it = timestamp_streams_data.stream_offsets.find(timestamp_stream_id); it != timestamp_streams_data.stream_offsets.end()) { - serialized_chunk.timestamps_offset = it->second; - return; - } - - timestamp_streams_data.stream_offsets.emplace(timestamp_stream_id, data_size); - } else { - if (const auto it = timestamp_streams_data.finalized_stream_offsets.find(timestamp_stream_id); - it != timestamp_streams_data.finalized_stream_offsets.end()) { - serialized_chunk.timestamps_offset = it->second; - return; - } - - timestamp_streams_data.finalized_stream_offsets.emplace(timestamp_stream_id, data_size); - } - - serialized_chunk.timestamps_offset = data_size; - data_size += storage_.get_timestamp_stream(timestamp_stream_id).stream.size_in_bytes(); - } - - template - void write_chunks_data(const ChunkList& chunks, TimestampStreamsData& timestamp_streams_data, std::ostream& stream) noexcept { - using enum chunk::DataChunk::Type; - - for (auto& chunk_data : chunks) { - if (chunk_data.is_open()) [[likely]] { - if (const auto& chunk = get_chunk(chunk_data); !chunk.is_empty()) [[likely]] { - write_chunk_data(chunk, timestamp_streams_data, stream); - } - } else { - write_chunk_data(get_chunk(chunk_data), timestamp_streams_data, stream); - } - } - } - - template - void write_chunk_data(const chunk::DataChunk& chunk, TimestampStreamsData& timestamp_streams_data, std::ostream& stream) noexcept { - using enum EncodingType; - - switch (chunk.encoding_state.encoding_type) { - case kUint32Constant: - case kFloat32Constant: { - write_timestamp_stream(timestamp_streams_data, chunk.timestamp_encoder_state_id, stream); - break; - } - - case kDoubleConstant: { - write_timestamp_stream(timestamp_streams_data, chunk.timestamp_encoder_state_id, stream); - stream.write(reinterpret_cast(&storage_.variant_encoders[chunk.encoder.external_index].double_constant), - sizeof(encoder::value::DoubleConstantEncoder)); - break; - } - - case kTwoDoubleConstant: { - write_timestamp_stream(timestamp_streams_data, chunk.timestamp_encoder_state_id, stream); - stream.write(reinterpret_cast(&storage_.variant_encoders[chunk.encoder.external_index].two_double_constant), - sizeof(encoder::value::TwoDoubleConstantEncoder)); - break; - } - - case kAscInteger: { - write_timestamp_stream(timestamp_streams_data, chunk.timestamp_encoder_state_id, stream); - write_compact_bit_sequence(storage_.get_asc_integer_stream(chunk.encoder.external_index), stream); - break; - } - - case kAscIntegerThenValuesGorilla: { - write_timestamp_stream(timestamp_streams_data, chunk.timestamp_encoder_state_id, stream); - write_compact_bit_sequence(storage_.get_asc_integer_then_values_gorilla_stream(chunk.encoder.external_index), stream); - break; - } - - case kValuesGorilla: { - write_timestamp_stream(timestamp_streams_data, chunk.timestamp_encoder_state_id, stream); - write_compact_bit_sequence(storage_.get_values_gorilla_stream(chunk.encoder.external_index), stream); - break; - } - - case kGorilla: { - write_compact_bit_sequence(storage_.get_gorilla_encoder_stream(chunk.encoder.external_index), stream); - break; - } - - default: { - assert(chunk.encoding_state.encoding_type != kUnknown); - break; - } - } - } - - template - PROMPP_ALWAYS_INLINE static void write_compact_bit_sequence(const CompactBitSequence& bit_sequence, std::ostream& stream) { - stream.write(reinterpret_cast(bit_sequence.raw_bytes()), bit_sequence.size_in_bytes()); - } - - template - PROMPP_ALWAYS_INLINE void write_timestamp_stream(TimestampStreamsData& timestamp_streams_data, - encoder::timestamp::State::Id stream_id, - std::ostream& stream) { - if constexpr (chunk_type == chunk::DataChunk::Type::kOpen) { - if (const auto it = timestamp_streams_data.stream_offsets.find(stream_id); it->second != TimestampStreamsData::kInvalidOffset) { - write_compact_bit_sequence(storage_.get_timestamp_stream(stream_id).stream, stream); - it->second = TimestampStreamsData::kInvalidOffset; - } - } else { - if (const auto it = timestamp_streams_data.finalized_stream_offsets.find(stream_id); it->second != TimestampStreamsData::kInvalidOffset) { - write_compact_bit_sequence(storage_.get_timestamp_stream(stream_id).stream, stream); - it->second = TimestampStreamsData::kInvalidOffset; - } - } - } -}; - -} // namespace series_data::serialization diff --git a/pp/series_data/tests/encoder/timestamp/encoder_tests.cpp b/pp/series_data/tests/encoder/timestamp/encoder_tests.cpp index c4c642fee7..75118dfedb 100644 --- a/pp/series_data/tests/encoder/timestamp/encoder_tests.cpp +++ b/pp/series_data/tests/encoder/timestamp/encoder_tests.cpp @@ -16,8 +16,8 @@ TEST_F(TimestampEncoderFixture, OneStateForTwoSeries) { // Arrange // Act - auto state_id1 = encoder_.encode(State::kInvalidId, 101); - auto state_id2 = encoder_.encode(State::kInvalidId, 101); + const auto state_id1 = encoder_.encode(State::kInvalidId, 101); + const auto state_id2 = encoder_.encode(State::kInvalidId, 101); // Assert EXPECT_EQ(0U, state_id1); @@ -30,8 +30,8 @@ TEST_F(TimestampEncoderFixture, TransitionToNewStateWithSavingPreviousState) { encoder_.encode(State::kInvalidId, 101); // Act - auto first_state_id = encoder_.encode(State::kInvalidId, 101); - auto state_id = encoder_.encode(first_state_id, 102); + const auto first_state_id = encoder_.encode(State::kInvalidId, 101); + const auto state_id = encoder_.encode(first_state_id, 102); // Assert EXPECT_EQ(0U, first_state_id); @@ -44,8 +44,8 @@ TEST_F(TimestampEncoderFixture, TransitionToNewStateWithErasingPreviousState) { // Arrange // Act - auto first_state_id = encoder_.encode(State::kInvalidId, 101); - auto state_id = encoder_.encode(first_state_id, 102); + const auto first_state_id = encoder_.encode(State::kInvalidId, 101); + const auto state_id = encoder_.encode(first_state_id, 102); // Assert EXPECT_EQ(0U, first_state_id); diff --git a/pp/series_data/tests/serialization/serializer_deserializer_new_tests.cpp b/pp/series_data/tests/serialization/serializer_deserializer_new_tests.cpp deleted file mode 100644 index c06c9b5f1f..0000000000 --- a/pp/series_data/tests/serialization/serializer_deserializer_new_tests.cpp +++ /dev/null @@ -1,1047 +0,0 @@ -#include - -#include "bare_bones/streams.h" -#include "series_data/data_storage.h" -#include "series_data/encoder.h" -#include "series_data/encoder/bit_sequence.h" -#include "series_data/serialization/deserializer.h" -#include "series_data/serialization/serialized_data.h" -#include "series_data/serialization/serializer.h" - -namespace { - -using BareBones::Encoding::Gorilla::STALE_NAN; -using series_data::ChunkFinalizer; -using series_data::DataStorage; -using series_data::Encoder; -using series_data::EncodingType; -using series_data::chunk::DataChunk; -using series_data::decoder::DecodeIteratorSentinel; -using series_data::encoder::Sample; -using series_data::encoder::SampleList; -using series_data::querier::QueriedChunk; -using series_data::querier::QueriedChunkList; -using series_data::serialization::DataSerializer; -using series_data::serialization::SerializedData; -using series_data::serialization::SerializedDataView; - -class SerializerDeserializerTrait { - protected: - DataStorage storage_; - Encoder<> encoder_{storage_}; - DataSerializer serializer_{storage_}; - - [[nodiscard]] PROMPP_ALWAYS_INLINE static SampleList decode_current_chunk(SerializedDataView& data, uint32_t series_id) { - SampleList result; - - EXPECT_EQ(series_id, data.next_series().first); - - std::ranges::copy(data.create_current_series_iterator(), DecodeIteratorSentinel{}, std::back_insert_iterator(result)); - - return result; - } - - [[nodiscard]] PROMPP_ALWAYS_INLINE static SampleList decode_chunk_by_id(const SerializedDataView& data, uint32_t series_chunk_id) { - SampleList result; - - std::ranges::copy(data.create_series_iterator(series_chunk_id), DecodeIteratorSentinel{}, std::back_insert_iterator(result)); - - return result; - } -}; - -class SerializerDeserializerFixtureNew : public SerializerDeserializerTrait, public testing::Test {}; - -TEST_F(SerializerDeserializerFixtureNew, EmptyChunksList) { - // Arrange - - // Act - const SerializedData serialized = serializer_.serialize({}); - const SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(0U, serialized_view.get_chunks_view().size()); - ASSERT_EQ(series_data::encoder::CompactBitSequence::reserved_bytes_for_reader().size(), serialized_view.get_buffer_view().size()); -} - -TEST_F(SerializerDeserializerFixtureNew, TwoUint32ConstantChunkWithCommonTimestampStream) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(1, 1, 1.0); - - encoder_.encode(0, 2, 1.0); - encoder_.encode(1, 2, 1.0); - - encoder_.encode(0, 3, 1.0); - encoder_.encode(1, 3, 1.0); - - // Act - const SerializedData serialized = serializer_.serialize({QueriedChunk{0}, QueriedChunk{1}}); - SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(2U, serialized_view.get_chunks_view().size()); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); - EXPECT_EQ(serialized_view.get_chunks_view()[0].timestamps_offset, serialized_view.get_chunks_view()[1].timestamps_offset); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 1, .value = 1.0}, - {.timestamp = 2, .value = 1.0}, - {.timestamp = 3, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 0))); - - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 1, .value = 1.0}, - {.timestamp = 2, .value = 1.0}, - {.timestamp = 3, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 1))); -} - -TEST_F(SerializerDeserializerFixtureNew, TwoUint32ConstantFinalizedChunkWithCommonTimestampStream) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(1, 1, 1.0); - - encoder_.encode(0, 2, 1.0); - encoder_.encode(1, 2, 1.0); - - encoder_.encode(0, 3, 1.0); - encoder_.encode(1, 3, 1.0); - - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); - encoder_.encode(0, 4, 1.0); - encoder_.encode(1, 4, 1.0); - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(4U, serialized_view.get_chunks_view().size()); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); - EXPECT_EQ(serialized_view.get_chunks_view()[0].timestamps_offset, serialized_view.get_chunks_view()[2].timestamps_offset); - EXPECT_EQ(serialized_view.get_chunks_view()[1].timestamps_offset, serialized_view.get_chunks_view()[3].timestamps_offset); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 1, .value = 1.0}, - {.timestamp = 2, .value = 1.0}, - {.timestamp = 3, .value = 1.0}, - {.timestamp = 4, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 0))); - - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 1, .value = 1.0}, - {.timestamp = 2, .value = 1.0}, - {.timestamp = 3, .value = 1.0}, - {.timestamp = 4, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 1))); -} - -TEST_F(SerializerDeserializerFixtureNew, ThreeUint32ConstantChunkWithCommonAndUniqueTimestampStream) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(1, 1, 1.0); - - encoder_.encode(0, 2, 1.0); - encoder_.encode(1, 2, 1.0); - - encoder_.encode(0, 3, 1.0); - encoder_.encode(1, 3, 1.0); - - encoder_.encode(2, 1, 2.0); - encoder_.encode(2, 2, 2.0); - encoder_.encode(2, 3, 2.0); - - // Act - const SerializedData serialized = serializer_.serialize({QueriedChunk{0}, QueriedChunk{1}, QueriedChunk{2}}); - SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(3U, serialized_view.get_chunks_view().size()); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); - EXPECT_EQ(serialized_view.get_chunks_view()[0].timestamps_offset, serialized_view.get_chunks_view()[1].timestamps_offset); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 1, .value = 1.0}, - {.timestamp = 2, .value = 1.0}, - {.timestamp = 3, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 0))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 1, .value = 1.0}, - {.timestamp = 2, .value = 1.0}, - {.timestamp = 3, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 1))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 1, .value = 2.0}, - {.timestamp = 2, .value = 2.0}, - {.timestamp = 3, .value = 2.0}, - }, - decode_current_chunk(serialized_view, 2))); -} - -TEST_F(SerializerDeserializerFixtureNew, AllChunkTypes) { - // Arrange - encoder_.encode(0, 100, 1.0); - - encoder_.encode(1, 101, 1.1); - - encoder_.encode(2, 102, 1.1); - encoder_.encode(2, 103, 1.2); - - encoder_.encode(3, 104, 1.0); - encoder_.encode(3, 105, 2.0); - encoder_.encode(3, 106, 3.0); - - encoder_.encode(4, 107, 1.1); - encoder_.encode(20, 107, 1.1); - encoder_.encode(4, 108, 2.1); - encoder_.encode(20, 108, 2.1); - encoder_.encode(4, 109, 3.1); - - encoder_.encode(5, 110, 1.1); - encoder_.encode(5, 111, 2.1); - encoder_.encode(5, 112, 3.1); - - encoder_.encode(6, 113, 2.0); - - encoder_.encode(7, 114, -1.0); - encoder_.encode(7, 115, -1.0); - - encoder_.encode(8, 120, 1.0); - encoder_.encode(8, 121, 2.0); - encoder_.encode(8, 122, 3.0); - encoder_.encode(8, 123, 4.1); - - // Act - SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); - - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 100, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 0))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 101, .value = 1.1}, - }, - decode_current_chunk(serialized_view, 1))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 102, .value = 1.1}, - {.timestamp = 103, .value = 1.2}, - }, - decode_current_chunk(serialized_view, 2))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 104, .value = 1.0}, - {.timestamp = 105, .value = 2.0}, - {.timestamp = 106, .value = 3.0}, - }, - decode_current_chunk(serialized_view, 3))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 107, .value = 1.1}, - {.timestamp = 108, .value = 2.1}, - {.timestamp = 109, .value = 3.1}, - }, - decode_current_chunk(serialized_view, 4))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 110, .value = 1.1}, - {.timestamp = 111, .value = 2.1}, - {.timestamp = 112, .value = 3.1}, - }, - decode_current_chunk(serialized_view, 5))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 113, .value = 2.0}, - }, - decode_current_chunk(serialized_view, 6))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 114, .value = -1.0}, - {.timestamp = 115, .value = -1.0}, - }, - decode_current_chunk(serialized_view, 7))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 120, .value = 1.0}, - {.timestamp = 121, .value = 2.0}, - {.timestamp = 122, .value = 3.0}, - {.timestamp = 123, .value = 4.1}, - }, - decode_current_chunk(serialized_view, 8))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 107, .value = 1.1}, - {.timestamp = 108, .value = 2.1}, - }, - decode_current_chunk(serialized_view, 20))); -} - -TEST_F(SerializerDeserializerFixtureNew, FinalizedAllChunkTypes) { - // Arrange - encoder_.encode(0, 100, 1.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - - encoder_.encode(1, 101, 1.1); - ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); - - encoder_.encode(2, 102, 1.1); - encoder_.encode(2, 103, 1.2); - ChunkFinalizer::finalize(storage_, 2, storage_.open_chunks[2]); - - encoder_.encode(3, 104, 1.0); - encoder_.encode(3, 105, 2.0); - encoder_.encode(3, 106, 3.0); - ChunkFinalizer::finalize(storage_, 3, storage_.open_chunks[3]); - - encoder_.encode(4, 107, 1.1); - encoder_.encode(20, 107, 1.1); - encoder_.encode(4, 108, 2.1); - encoder_.encode(20, 108, 2.1); - encoder_.encode(4, 109, 3.1); - ChunkFinalizer::finalize(storage_, 4, storage_.open_chunks[4]); - ChunkFinalizer::finalize(storage_, 20, storage_.open_chunks[20]); - - encoder_.encode(5, 110, 1.1); - encoder_.encode(5, 111, 2.1); - encoder_.encode(5, 112, 3.1); - ChunkFinalizer::finalize(storage_, 5, storage_.open_chunks[5]); - - encoder_.encode(6, 113, 2.0); - ChunkFinalizer::finalize(storage_, 6, storage_.open_chunks[6]); - - encoder_.encode(7, 114, -1.0); - encoder_.encode(7, 115, -1.0); - ChunkFinalizer::finalize(storage_, 7, storage_.open_chunks[7]); - - encoder_.encode(8, 120, 1.0); - encoder_.encode(8, 121, 2.0); - encoder_.encode(8, 122, 3.0); - encoder_.encode(8, 123, 4.1); - ChunkFinalizer::finalize(storage_, 8, storage_.open_chunks[8]); - - // Act - SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); - - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 100, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 0))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 101, .value = 1.1}, - }, - decode_current_chunk(serialized_view, 1))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 102, .value = 1.1}, - {.timestamp = 103, .value = 1.2}, - }, - decode_current_chunk(serialized_view, 2))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 104, .value = 1.0}, - {.timestamp = 105, .value = 2.0}, - {.timestamp = 106, .value = 3.0}, - }, - decode_current_chunk(serialized_view, 3))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 107, .value = 1.1}, - {.timestamp = 108, .value = 2.1}, - {.timestamp = 109, .value = 3.1}, - }, - decode_current_chunk(serialized_view, 4))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 110, .value = 1.1}, - {.timestamp = 111, .value = 2.1}, - {.timestamp = 112, .value = 3.1}, - }, - decode_current_chunk(serialized_view, 5))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 113, .value = 2.0}, - }, - decode_current_chunk(serialized_view, 6))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 114, .value = -1.0}, - {.timestamp = 115, .value = -1.0}, - }, - decode_current_chunk(serialized_view, 7))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 120, .value = 1.0}, - {.timestamp = 121, .value = 2.0}, - {.timestamp = 122, .value = 3.0}, - {.timestamp = 123, .value = 4.1}, - }, - decode_current_chunk(serialized_view, 8))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 107, .value = 1.1}, - {.timestamp = 108, .value = 2.1}, - }, - decode_current_chunk(serialized_view, 20))); -} - -TEST_F(SerializerDeserializerFixtureNew, ChunkWithFinalizedTimestampStream) { - // Arrange - encoder_.encode(0, 100, 1.0); - encoder_.encode(1, 100, 1.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - - // Act - const SerializedData serialized = serializer_.serialize({QueriedChunk{1}}); - SerializedDataView serialized_view(serialized); - - // Assert - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 100, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 1))); -} - -TEST_F(SerializerDeserializerFixtureNew, MultipleChunksOnOneSeriesId) { - // Arrange - encoder_.encode(0, 100, 1.0); - encoder_.encode(0, 101, 1.0); - encoder_.encode(0, 102, 1.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 103, 1.0); - encoder_.encode(0, 104, 1.0); - encoder_.encode(0, 105, 1.0); - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 100, .value = 1.0}, - {.timestamp = 101, .value = 1.0}, - {.timestamp = 102, .value = 1.0}, - {.timestamp = 103, .value = 1.0}, - {.timestamp = 104, .value = 1.0}, - {.timestamp = 105, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 0))); -} - -TEST_F(SerializerDeserializerFixtureNew, QueryFinalizedOnly) { - // Arrange - encoder_.encode(0, 100, 1.0); - encoder_.encode(0, 101, 1.0); - encoder_.encode(0, 102, 1.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 103, 1.0); - encoder_.encode(0, 104, 1.0); - encoder_.encode(0, 105, 1.0); - - // Act - const SerializedData serialized = serializer_.serialize({QueriedChunk{0, 0}}); - SerializedDataView serialized_view(serialized); - - // Assert - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 100, .value = 1.0}, - {.timestamp = 101, .value = 1.0}, - {.timestamp = 102, .value = 1.0}, - }, - decode_current_chunk(serialized_view, 0))); -} - -TEST_F(SerializerDeserializerFixtureNew, MultipleChunksOnOneSeriesIdWithSeveralFinalized) { - // Arrange - encoder_.encode(0, 100, 1.0); - encoder_.encode(0, 101, 2.0); - encoder_.encode(0, 102, 3.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 103, 4.0); - encoder_.encode(0, 104, 5.0); - encoder_.encode(0, 105, 6.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 106, 7.0); - encoder_.encode(0, 107, 8.0); - encoder_.encode(0, 108, 9.0); - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - EXPECT_TRUE(std::ranges::equal(SampleList{{.timestamp = 100, .value = 1.0}, - {.timestamp = 101, .value = 2.0}, - {.timestamp = 102, .value = 3.0}, - {.timestamp = 103, .value = 4.0}, - {.timestamp = 104, .value = 5.0}, - {.timestamp = 105, .value = 6.0}, - {.timestamp = 106, .value = 7.0}, - {.timestamp = 107, .value = 8.0}, - {.timestamp = 108, .value = 9.0}}, - decode_current_chunk(serialized_view, 0))); -} - -TEST_F(SerializerDeserializerFixtureNew, CreateIteratorFromChunkId) { - // Arrange - encoder_.encode(0, 100, 1.0); - encoder_.encode(0, 101, 2.0); - encoder_.encode(0, 102, 3.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 103, 4.0); - encoder_.encode(0, 104, 5.0); - encoder_.encode(0, 105, 6.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 106, 7.0); - encoder_.encode(0, 107, 8.0); - encoder_.encode(0, 108, 9.0); - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - EXPECT_TRUE(std::ranges::equal(SampleList{{.timestamp = 100, .value = 1.0}, - {.timestamp = 101, .value = 2.0}, - {.timestamp = 102, .value = 3.0}, - {.timestamp = 103, .value = 4.0}, - {.timestamp = 104, .value = 5.0}, - {.timestamp = 105, .value = 6.0}, - {.timestamp = 106, .value = 7.0}, - {.timestamp = 107, .value = 8.0}, - {.timestamp = 108, .value = 9.0}}, - decode_chunk_by_id(serialized_view, serialized_view.next_series().second))); -} - -TEST_F(SerializerDeserializerFixtureNew, AllChunkTypesWithStalenan) { - // Arrange - encoder_.encode(0, 100, 1.0); - encoder_.encode(0, 101, STALE_NAN); - - encoder_.encode(1, 102, 1.1); - encoder_.encode(1, 103, STALE_NAN); - - encoder_.encode(2, 104, 1.1); - encoder_.encode(2, 105, 1.2); - encoder_.encode(2, 106, STALE_NAN); - - encoder_.encode(3, 107, 1.0); - encoder_.encode(3, 108, 2.0); - encoder_.encode(3, 109, 3.0); - encoder_.encode(3, 110, STALE_NAN); - - encoder_.encode(4, 111, 1.1); - encoder_.encode(20, 111, 1.1); - encoder_.encode(4, 112, 2.1); - encoder_.encode(20, 112, 2.1); - encoder_.encode(4, 113, 3.1); - encoder_.encode(4, 114, STALE_NAN); - encoder_.encode(20, 113, STALE_NAN); - - encoder_.encode(5, 115, 1.1); - encoder_.encode(5, 116, 2.1); - encoder_.encode(5, 117, 3.1); - encoder_.encode(5, 118, STALE_NAN); - - encoder_.encode(6, 119, 2.0); - encoder_.encode(6, 120, STALE_NAN); - - encoder_.encode(7, 121, -1.0); - encoder_.encode(7, 122, -1.0); - encoder_.encode(7, 123, STALE_NAN); - - encoder_.encode(8, 130, 1.0); - encoder_.encode(8, 131, 2.0); - encoder_.encode(8, 132, 3.0); - encoder_.encode(8, 133, 4.1); - encoder_.encode(8, 134, STALE_NAN); - - // Act - SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); - EXPECT_TRUE(std::ranges::all_of(serialized_view.get_chunks_view(), [](const auto& chunk) { return chunk.encoding_state.has_last_stalenan; })); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); - - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 100, .value = 1.0}, - {.timestamp = 101, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 0))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 102, .value = 1.1}, - {.timestamp = 103, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 1))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 104, .value = 1.1}, - {.timestamp = 105, .value = 1.2}, - {.timestamp = 106, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 2))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 107, .value = 1.0}, - {.timestamp = 108, .value = 2.0}, - {.timestamp = 109, .value = 3.0}, - {.timestamp = 110, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 3))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 111, .value = 1.1}, - {.timestamp = 112, .value = 2.1}, - {.timestamp = 113, .value = 3.1}, - {.timestamp = 114, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 4))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 115, .value = 1.1}, - {.timestamp = 116, .value = 2.1}, - {.timestamp = 117, .value = 3.1}, - {.timestamp = 118, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 5))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 119, .value = 2.0}, - {.timestamp = 120, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 6))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 121, .value = -1.0}, - {.timestamp = 122, .value = -1.0}, - {.timestamp = 123, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 7))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 130, .value = 1.0}, - {.timestamp = 131, .value = 2.0}, - {.timestamp = 132, .value = 3.0}, - {.timestamp = 133, .value = 4.1}, - {.timestamp = 134, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 8))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 111, .value = 1.1}, - {.timestamp = 112, .value = 2.1}, - {.timestamp = 113, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 20))); -} - -TEST_F(SerializerDeserializerFixtureNew, FinalizedAllChunkTypesWithStalenan) { - // Arrange - encoder_.encode(0, 100, 1.0); - encoder_.encode(0, 101, STALE_NAN); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - - encoder_.encode(1, 102, 1.1); - encoder_.encode(1, 103, STALE_NAN); - ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); - - encoder_.encode(2, 104, 1.1); - encoder_.encode(2, 105, 1.2); - encoder_.encode(2, 106, STALE_NAN); - ChunkFinalizer::finalize(storage_, 2, storage_.open_chunks[2]); - - encoder_.encode(3, 107, 1.0); - encoder_.encode(3, 108, 2.0); - encoder_.encode(3, 109, 3.0); - encoder_.encode(3, 110, STALE_NAN); - ChunkFinalizer::finalize(storage_, 3, storage_.open_chunks[3]); - - encoder_.encode(4, 111, 1.1); - encoder_.encode(20, 111, 1.1); - encoder_.encode(4, 112, 2.1); - encoder_.encode(20, 112, 2.1); - encoder_.encode(4, 113, 3.1); - encoder_.encode(4, 114, STALE_NAN); - encoder_.encode(20, 113, STALE_NAN); - ChunkFinalizer::finalize(storage_, 4, storage_.open_chunks[4]); - ChunkFinalizer::finalize(storage_, 20, storage_.open_chunks[20]); - - encoder_.encode(5, 115, 1.1); - encoder_.encode(5, 116, 2.1); - encoder_.encode(5, 117, 3.1); - encoder_.encode(5, 118, STALE_NAN); - ChunkFinalizer::finalize(storage_, 5, storage_.open_chunks[5]); - - encoder_.encode(6, 119, 2.0); - encoder_.encode(6, 120, STALE_NAN); - ChunkFinalizer::finalize(storage_, 6, storage_.open_chunks[6]); - - encoder_.encode(7, 121, -1.0); - encoder_.encode(7, 122, -1.0); - encoder_.encode(7, 123, STALE_NAN); - ChunkFinalizer::finalize(storage_, 7, storage_.open_chunks[7]); - - encoder_.encode(8, 130, 1.0); - encoder_.encode(8, 131, 2.0); - encoder_.encode(8, 132, 3.0); - encoder_.encode(8, 133, 4.1); - encoder_.encode(8, 134, STALE_NAN); - ChunkFinalizer::finalize(storage_, 8, storage_.open_chunks[8]); - - // Act - SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - // Assert - ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); - EXPECT_TRUE(std::ranges::all_of(serialized_view.get_chunks_view(), [](const auto& chunk) { return chunk.encoding_state.has_last_stalenan; })); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); - - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 100, .value = 1.0}, - {.timestamp = 101, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 0))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 102, .value = 1.1}, - {.timestamp = 103, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 1))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 104, .value = 1.1}, - {.timestamp = 105, .value = 1.2}, - {.timestamp = 106, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 2))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 107, .value = 1.0}, - {.timestamp = 108, .value = 2.0}, - {.timestamp = 109, .value = 3.0}, - {.timestamp = 110, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 3))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 111, .value = 1.1}, - {.timestamp = 112, .value = 2.1}, - {.timestamp = 113, .value = 3.1}, - {.timestamp = 114, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 4))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 115, .value = 1.1}, - {.timestamp = 116, .value = 2.1}, - {.timestamp = 117, .value = 3.1}, - {.timestamp = 118, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 5))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 119, .value = 2.0}, - {.timestamp = 120, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 6))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 121, .value = -1.0}, - {.timestamp = 122, .value = -1.0}, - {.timestamp = 123, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 7))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 130, .value = 1.0}, - {.timestamp = 131, .value = 2.0}, - {.timestamp = 132, .value = 3.0}, - {.timestamp = 133, .value = 4.1}, - {.timestamp = 134, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 8))); - EXPECT_TRUE(std::ranges::equal( - SampleList{ - {.timestamp = 111, .value = 1.1}, - {.timestamp = 112, .value = 2.1}, - {.timestamp = 113, .value = STALE_NAN}, - }, - decode_current_chunk(serialized_view, 20))); -} - -class SerializedDataNextIterFixture : public SerializerDeserializerTrait, public testing::Test { - protected: - static std::vector get_chunks_ids(SerializedDataView& view) { - std::vector ans{}; - uint32_t id = view.next_series().first; - while (id != SerializedDataView::kNoMoreSeries) { - ans.push_back(id); - id = view.next_series().first; - } - return ans; - } -}; - -TEST_F(SerializedDataNextIterFixture, EmptyChunksList) { - // Arrange - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - const auto ids = get_chunks_ids(serialized_view); - - // Assert - EXPECT_TRUE(ids.empty()); - EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); -} - -TEST_F(SerializedDataNextIterFixture, OneChunk) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(0, 2, 1.0); - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - auto ids = get_chunks_ids(serialized_view); - - // Assert - EXPECT_TRUE(std::ranges::equal(ids, std::initializer_list{0u})); - EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); -} - -TEST_F(SerializedDataNextIterFixture, OneChunkFinalized) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(0, 2, 1.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 3, 1.0); - encoder_.encode(0, 4, 1.0); - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - auto ids = get_chunks_ids(serialized_view); - - // Assert - EXPECT_TRUE(std::ranges::equal(ids, std::initializer_list{0u})); - EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); -} - -TEST_F(SerializedDataNextIterFixture, SeveralChunks) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(1, 1, 1.0); - - encoder_.encode(0, 2, 1.0); - encoder_.encode(1, 2, 1.0); - - encoder_.encode(0, 3, 1.0); - encoder_.encode(1, 3, 1.0); - - encoder_.encode(2, 1, 2.0); - encoder_.encode(2, 2, 2.0); - encoder_.encode(2, 3, 2.0); - - encoder_.encode(100, 4, 2.1); - encoder_.encode(100, 5, 2.2); - encoder_.encode(100, 7, 2.3); - - // Act - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - auto ids = get_chunks_ids(serialized_view); - - // Assert - EXPECT_TRUE(std::ranges::equal(ids, std::initializer_list{0u, 1u, 2u, 100u})); - EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); -} - -class SerializedDataIterFixture : public SerializerDeserializerTrait, public testing::Test {}; - -TEST_F(SerializedDataIterFixture, ResetIteratorToSameSeries) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(0, 2, 1.0); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - encoder_.encode(0, 3, 1.0); - encoder_.encode(0, 4, 1.0); - - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - auto [series_id, chunk_id] = serialized_view.next_series(); - - // Act - auto iter = serialized_view.create_series_iterator(chunk_id); - iter.reset(serialized_view.get_buffer_view(), serialized_view.get_chunks_view(), chunk_id); - - // Assert - EXPECT_EQ(series_id, 0u); - EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); - EXPECT_TRUE(std::ranges::equal(std::ranges::subrange(iter, DecodeIteratorSentinel{}), - std::initializer_list{Sample{.timestamp = 1, .value = 1.0}, Sample{.timestamp = 2, .value = 1.0}, - Sample{.timestamp = 3, .value = 1.0}, Sample{.timestamp = 4, .value = 1.0}})); -} - -TEST_F(SerializedDataIterFixture, ResetIteratorToAnotherSeries) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(1, 1, 1.0); - - encoder_.encode(0, 2, 1.0); - encoder_.encode(1, 2, 1.1); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); - - encoder_.encode(0, 3, 1.0); - encoder_.encode(1, 3, 1.2); - - encoder_.encode(0, 4, 1.0); - encoder_.encode(1, 4, 1.3); - - const SerializedData serialized = serializer_.serialize(); - SerializedDataView serialized_view(serialized); - - auto [series_id0, chunk_id0] = serialized_view.next_series(); - auto [series_id1, chunk_id1] = serialized_view.next_series(); - - // Act - auto iter = serialized_view.create_series_iterator(chunk_id0); - iter.reset(serialized_view.get_buffer_view(), serialized_view.get_chunks_view(), chunk_id1); - - // Assert - EXPECT_EQ(series_id0, 0u); - EXPECT_EQ(series_id1, 1u); - EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); - EXPECT_TRUE(std::ranges::equal(std::ranges::subrange(iter, DecodeIteratorSentinel{}), - std::initializer_list{Sample{.timestamp = 1, .value = 1.0}, Sample{.timestamp = 2, .value = 1.1}, - Sample{.timestamp = 3, .value = 1.2}, Sample{.timestamp = 4, .value = 1.3}})); -} - -TEST_F(SerializedDataIterFixture, ResetIteratorToAnotherSerializedData) { - // Arrange - encoder_.encode(0, 1, 1.0); - encoder_.encode(1, 1, 1.0); - - encoder_.encode(0, 2, 1.0); - encoder_.encode(1, 2, 1.1); - ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); - ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); - - encoder_.encode(0, 3, 1.0); - encoder_.encode(1, 3, 1.2); - - encoder_.encode(0, 4, 1.0); - encoder_.encode(1, 4, 1.3); - - const SerializedData serialized0 = serializer_.serialize(); - SerializedDataView serialized_view0(serialized0); - - encoder_.encode(0, 5, 1.0); - encoder_.encode(1, 5, 1.4); - - const SerializedData serialized1 = serializer_.serialize(); - SerializedDataView serialized_view1(serialized1); - - auto [series_id0, chunk_id0] = serialized_view0.next_series(); - - std::ignore = serialized_view1.next_series(); - auto [series_id1, chunk_id1] = serialized_view1.next_series(); - - // Act - auto iter = serialized_view0.create_series_iterator(chunk_id0); - iter.reset(serialized_view1.get_buffer_view(), serialized_view1.get_chunks_view(), chunk_id1); - - // Assert - EXPECT_TRUE( - std::ranges::equal(std::ranges::subrange(iter, DecodeIteratorSentinel{}), - std::initializer_list{Sample{.timestamp = 1, .value = 1.0}, Sample{.timestamp = 2, .value = 1.1}, Sample{.timestamp = 3, .value = 1.2}, - Sample{.timestamp = 4, .value = 1.3}, Sample{.timestamp = 5, .value = 1.4}})); -} - -} // namespace \ No newline at end of file diff --git a/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp b/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp index 15b5d52ebd..132b3276c1 100644 --- a/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp +++ b/pp/series_data/tests/serialization/serializer_deserializer_tests.cpp @@ -3,8 +3,9 @@ #include "bare_bones/streams.h" #include "series_data/data_storage.h" #include "series_data/encoder.h" +#include "series_data/encoder/bit_sequence.h" #include "series_data/serialization/deserializer.h" -#include "series_data/serialization/serializer.h" +#include "series_data/serialization/serialized_data.h" namespace { @@ -19,26 +20,41 @@ using series_data::encoder::Sample; using series_data::encoder::SampleList; using series_data::querier::QueriedChunk; using series_data::querier::QueriedChunkList; -using series_data::serialization::Deserializer; -using series_data::serialization::Serializer; +using series_data::serialization::DataSerializer; +using series_data::serialization::SerializedData; +using series_data::serialization::SerializedDataView; class SerializerDeserializerTrait { protected: DataStorage storage_; - Serializer serializer_{storage_}; Encoder<> encoder_{storage_}; - BareBones::ShrinkedToFitOStringStream stream_; + DataSerializer serializer_{storage_}; - [[nodiscard]] PROMPP_ALWAYS_INLINE std::span get_buffer() const noexcept { - return {reinterpret_cast(stream_.view().data()), stream_.view().size()}; + [[nodiscard]] PROMPP_ALWAYS_INLINE static SampleList decode_current_chunk(SerializedDataView& data, uint32_t series_id) { + EXPECT_EQ(series_id, data.next_series().first); + + SampleList result; + std::ranges::copy(data.create_current_series_iterator(), DecodeIteratorSentinel{}, std::back_insert_iterator(result)); + return result; } - template - [[nodiscard]] PROMPP_ALWAYS_INLINE static SampleList decode_chunk(DecodeIterator iterator) { + [[nodiscard]] PROMPP_ALWAYS_INLINE static SampleList decode_chunk_by_id(const SerializedDataView& data, uint32_t series_chunk_id) { SampleList result; - std::ranges::copy(iterator, DecodeIteratorSentinel{}, std::back_insert_iterator(result)); + std::ranges::copy(data.create_series_iterator(series_chunk_id), DecodeIteratorSentinel{}, std::back_insert_iterator(result)); return result; } + + [[nodiscard]] PROMPP_ALWAYS_INLINE SerializedData serialize(const QueriedChunkList& queried_chunks) noexcept { + auto data = serializer_.serialize(queried_chunks); + storage_.reset(); + return data; + } + + [[nodiscard]] PROMPP_ALWAYS_INLINE SerializedData serialize() noexcept { + auto data = serializer_.serialize(); + storage_.reset(); + return data; + } }; class SerializerDeserializerFixture : public SerializerDeserializerTrait, public testing::Test {}; @@ -47,12 +63,12 @@ TEST_F(SerializerDeserializerFixture, EmptyChunksList) { // Arrange // Act - serializer_.serialize({}, stream_); - const Deserializer deserializer(get_buffer()); + const auto serialized = serialize({}); + const SerializedDataView serialized_view(serialized); // Assert - ASSERT_TRUE(deserializer.is_valid()); - ASSERT_EQ(0U, deserializer.get_chunks().size()); + ASSERT_EQ(0U, serialized_view.get_chunks_view().size()); + ASSERT_EQ(series_data::encoder::CompactBitSequence::reserved_bytes_for_reader().size(), serialized_view.get_buffer_view().size()); } TEST_F(SerializerDeserializerFixture, TwoUint32ConstantChunkWithCommonTimestampStream) { @@ -67,29 +83,76 @@ TEST_F(SerializerDeserializerFixture, TwoUint32ConstantChunkWithCommonTimestampS encoder_.encode(1, 3, 1.0); // Act - serializer_.serialize({QueriedChunk{0}, QueriedChunk{1}}, stream_); - const Deserializer deserializer(get_buffer()); + const auto serialized = serialize({QueriedChunk{0}, QueriedChunk{1}}); + SerializedDataView serialized_view(serialized); + + // Assert + ASSERT_EQ(2U, serialized_view.get_chunks_view().size()); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); + EXPECT_EQ(serialized_view.get_chunks_view()[0].timestamps_offset, serialized_view.get_chunks_view()[1].timestamps_offset); + EXPECT_TRUE(std::ranges::equal( + SampleList{ + {.timestamp = 1, .value = 1.0}, + {.timestamp = 2, .value = 1.0}, + {.timestamp = 3, .value = 1.0}, + }, + decode_current_chunk(serialized_view, 0))); + + EXPECT_TRUE(std::ranges::equal( + SampleList{ + {.timestamp = 1, .value = 1.0}, + {.timestamp = 2, .value = 1.0}, + {.timestamp = 3, .value = 1.0}, + }, + decode_current_chunk(serialized_view, 1))); +} + +TEST_F(SerializerDeserializerFixture, TwoUint32ConstantFinalizedChunkWithCommonTimestampStream) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(1, 1, 1.0); + + encoder_.encode(0, 2, 1.0); + encoder_.encode(1, 2, 1.0); + + encoder_.encode(0, 3, 1.0); + encoder_.encode(1, 3, 1.0); + + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); + encoder_.encode(0, 4, 1.0); + encoder_.encode(1, 4, 1.0); + + // Act + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); // Assert - ASSERT_TRUE(deserializer.is_valid()); - ASSERT_EQ(2U, deserializer.get_chunks().size()); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[1].encoding_state.encoding_type); - EXPECT_EQ(deserializer.get_chunks()[0].timestamps_offset, deserializer.get_chunks()[1].timestamps_offset); + ASSERT_EQ(4U, serialized_view.get_chunks_view().size()); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); + EXPECT_EQ(serialized_view.get_chunks_view()[0].timestamps_offset, serialized_view.get_chunks_view()[2].timestamps_offset); + EXPECT_EQ(serialized_view.get_chunks_view()[1].timestamps_offset, serialized_view.get_chunks_view()[3].timestamps_offset); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 1, .value = 1.0}, {.timestamp = 2, .value = 1.0}, {.timestamp = 3, .value = 1.0}, + {.timestamp = 4, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 0))); + EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 1, .value = 1.0}, {.timestamp = 2, .value = 1.0}, {.timestamp = 3, .value = 1.0}, + {.timestamp = 4, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[1])))); + decode_current_chunk(serialized_view, 1))); } TEST_F(SerializerDeserializerFixture, ThreeUint32ConstantChunkWithCommonAndUniqueTimestampStream) { @@ -108,37 +171,36 @@ TEST_F(SerializerDeserializerFixture, ThreeUint32ConstantChunkWithCommonAndUniqu encoder_.encode(2, 3, 2.0); // Act - serializer_.serialize({QueriedChunk{0}, QueriedChunk{1}, QueriedChunk{2}}, stream_); - const Deserializer deserializer(get_buffer()); + const auto serialized = serialize({QueriedChunk{0}, QueriedChunk{1}, QueriedChunk{2}}); + SerializedDataView serialized_view(serialized); // Assert - ASSERT_TRUE(deserializer.is_valid()); - ASSERT_EQ(3U, deserializer.get_chunks().size()); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[2].encoding_state.encoding_type); - EXPECT_EQ(deserializer.get_chunks()[0].timestamps_offset, deserializer.get_chunks()[1].timestamps_offset); + ASSERT_EQ(3U, serialized_view.get_chunks_view().size()); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); + EXPECT_EQ(serialized_view.get_chunks_view()[0].timestamps_offset, serialized_view.get_chunks_view()[1].timestamps_offset); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 1, .value = 1.0}, {.timestamp = 2, .value = 1.0}, {.timestamp = 3, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 0))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 1, .value = 1.0}, {.timestamp = 2, .value = 1.0}, {.timestamp = 3, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[1])))); + decode_current_chunk(serialized_view, 1))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 1, .value = 2.0}, {.timestamp = 2, .value = 2.0}, {.timestamp = 3, .value = 2.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[2])))); + decode_current_chunk(serialized_view, 2))); } TEST_F(SerializerDeserializerFixture, AllChunkTypes) { @@ -175,72 +237,71 @@ TEST_F(SerializerDeserializerFixture, AllChunkTypes) { encoder_.encode(8, 123, 4.1); // Act - serializer_.serialize(stream_); - Deserializer deserializer(get_buffer()); + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); // Assert - ASSERT_TRUE(deserializer.is_valid()); - ASSERT_EQ(10U, deserializer.get_chunks().size()); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, deserializer.get_chunks()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, deserializer.get_chunks()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, deserializer.get_chunks()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, deserializer.get_chunks()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, deserializer.get_chunks()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, deserializer.get_chunks()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, deserializer.get_chunks()[9].label_set_id); + ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); + ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 100, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 0))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 101, .value = 1.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[1])))); + decode_current_chunk(serialized_view, 1))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 102, .value = 1.1}, {.timestamp = 103, .value = 1.2}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[2])))); + decode_current_chunk(serialized_view, 2))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 104, .value = 1.0}, {.timestamp = 105, .value = 2.0}, {.timestamp = 106, .value = 3.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[3])))); + decode_current_chunk(serialized_view, 3))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 107, .value = 1.1}, {.timestamp = 108, .value = 2.1}, {.timestamp = 109, .value = 3.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[4])))); + decode_current_chunk(serialized_view, 4))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 110, .value = 1.1}, {.timestamp = 111, .value = 2.1}, {.timestamp = 112, .value = 3.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[5])))); + decode_current_chunk(serialized_view, 5))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 113, .value = 2.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[6])))); + decode_current_chunk(serialized_view, 6))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 114, .value = -1.0}, {.timestamp = 115, .value = -1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[7])))); + decode_current_chunk(serialized_view, 7))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 120, .value = 1.0}, @@ -248,13 +309,13 @@ TEST_F(SerializerDeserializerFixture, AllChunkTypes) { {.timestamp = 122, .value = 3.0}, {.timestamp = 123, .value = 4.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[8])))); + decode_current_chunk(serialized_view, 8))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 107, .value = 1.1}, {.timestamp = 108, .value = 2.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[9])))); + decode_current_chunk(serialized_view, 20))); } TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypes) { @@ -301,72 +362,71 @@ TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypes) { ChunkFinalizer::finalize(storage_, 8, storage_.open_chunks[8]); // Act - serializer_.serialize(stream_); - Deserializer deserializer(get_buffer()); + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); // Assert - ASSERT_TRUE(deserializer.is_valid()); - ASSERT_EQ(10U, deserializer.get_chunks().size()); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, deserializer.get_chunks()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, deserializer.get_chunks()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, deserializer.get_chunks()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, deserializer.get_chunks()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, deserializer.get_chunks()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, deserializer.get_chunks()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, deserializer.get_chunks()[9].label_set_id); + ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); + ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 100, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 0))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 101, .value = 1.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[1])))); + decode_current_chunk(serialized_view, 1))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 102, .value = 1.1}, {.timestamp = 103, .value = 1.2}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[2])))); + decode_current_chunk(serialized_view, 2))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 104, .value = 1.0}, {.timestamp = 105, .value = 2.0}, {.timestamp = 106, .value = 3.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[3])))); + decode_current_chunk(serialized_view, 3))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 107, .value = 1.1}, {.timestamp = 108, .value = 2.1}, {.timestamp = 109, .value = 3.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[4])))); + decode_current_chunk(serialized_view, 4))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 110, .value = 1.1}, {.timestamp = 111, .value = 2.1}, {.timestamp = 112, .value = 3.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[5])))); + decode_current_chunk(serialized_view, 5))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 113, .value = 2.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[6])))); + decode_current_chunk(serialized_view, 6))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 114, .value = -1.0}, {.timestamp = 115, .value = -1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[7])))); + decode_current_chunk(serialized_view, 7))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 120, .value = 1.0}, @@ -374,13 +434,13 @@ TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypes) { {.timestamp = 122, .value = 3.0}, {.timestamp = 123, .value = 4.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[8])))); + decode_current_chunk(serialized_view, 8))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 107, .value = 1.1}, {.timestamp = 108, .value = 2.1}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[9])))); + decode_current_chunk(serialized_view, 20))); } TEST_F(SerializerDeserializerFixture, ChunkWithFinalizedTimestampStream) { @@ -390,15 +450,15 @@ TEST_F(SerializerDeserializerFixture, ChunkWithFinalizedTimestampStream) { ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); // Act - serializer_.serialize({QueriedChunk{1}}, stream_); - const Deserializer deserializer(get_buffer()); + const auto serialized = serialize({QueriedChunk{1}}); + SerializedDataView serialized_view(serialized); // Assert EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 100, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 1))); } TEST_F(SerializerDeserializerFixture, MultipleChunksOnOneSeriesId) { @@ -408,10 +468,12 @@ TEST_F(SerializerDeserializerFixture, MultipleChunksOnOneSeriesId) { encoder_.encode(0, 102, 1.0); ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); encoder_.encode(0, 103, 1.0); + encoder_.encode(0, 104, 1.0); + encoder_.encode(0, 105, 1.0); // Act - serializer_.serialize(stream_); - const Deserializer deserializer(get_buffer()); + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); // Assert EXPECT_TRUE(std::ranges::equal( @@ -419,13 +481,97 @@ TEST_F(SerializerDeserializerFixture, MultipleChunksOnOneSeriesId) { {.timestamp = 100, .value = 1.0}, {.timestamp = 101, .value = 1.0}, {.timestamp = 102, .value = 1.0}, + {.timestamp = 103, .value = 1.0}, + {.timestamp = 104, .value = 1.0}, + {.timestamp = 105, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 0))); +} + +TEST_F(SerializerDeserializerFixture, QueryFinalizedOnly) { + // Arrange + encoder_.encode(0, 100, 1.0); + encoder_.encode(0, 101, 1.0); + encoder_.encode(0, 102, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 103, 1.0); + encoder_.encode(0, 104, 1.0); + encoder_.encode(0, 105, 1.0); + + // Act + const auto serialized = serialize({QueriedChunk{0, 0}}); + SerializedDataView serialized_view(serialized); + + // Assert EXPECT_TRUE(std::ranges::equal( SampleList{ - {.timestamp = 103, .value = 1.0}, + {.timestamp = 100, .value = 1.0}, + {.timestamp = 101, .value = 1.0}, + {.timestamp = 102, .value = 1.0}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[1])))); + decode_current_chunk(serialized_view, 0))); +} + +TEST_F(SerializerDeserializerFixture, MultipleChunksOnOneSeriesIdWithSeveralFinalized) { + // Arrange + encoder_.encode(0, 100, 1.0); + encoder_.encode(0, 101, 2.0); + encoder_.encode(0, 102, 3.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 103, 4.0); + encoder_.encode(0, 104, 5.0); + encoder_.encode(0, 105, 6.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 106, 7.0); + encoder_.encode(0, 107, 8.0); + encoder_.encode(0, 108, 9.0); + + // Act + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + // Assert + EXPECT_TRUE(std::ranges::equal(SampleList{{.timestamp = 100, .value = 1.0}, + {.timestamp = 101, .value = 2.0}, + {.timestamp = 102, .value = 3.0}, + {.timestamp = 103, .value = 4.0}, + {.timestamp = 104, .value = 5.0}, + {.timestamp = 105, .value = 6.0}, + {.timestamp = 106, .value = 7.0}, + {.timestamp = 107, .value = 8.0}, + {.timestamp = 108, .value = 9.0}}, + decode_current_chunk(serialized_view, 0))); +} + +TEST_F(SerializerDeserializerFixture, CreateIteratorFromChunkId) { + // Arrange + encoder_.encode(0, 100, 1.0); + encoder_.encode(0, 101, 2.0); + encoder_.encode(0, 102, 3.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 103, 4.0); + encoder_.encode(0, 104, 5.0); + encoder_.encode(0, 105, 6.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 106, 7.0); + encoder_.encode(0, 107, 8.0); + encoder_.encode(0, 108, 9.0); + + // Act + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + // Assert + EXPECT_TRUE(std::ranges::equal(SampleList{{.timestamp = 100, .value = 1.0}, + {.timestamp = 101, .value = 2.0}, + {.timestamp = 102, .value = 3.0}, + {.timestamp = 103, .value = 4.0}, + {.timestamp = 104, .value = 5.0}, + {.timestamp = 105, .value = 6.0}, + {.timestamp = 106, .value = 7.0}, + {.timestamp = 107, .value = 8.0}, + {.timestamp = 108, .value = 9.0}}, + decode_chunk_by_id(serialized_view, serialized_view.next_series().second))); } TEST_F(SerializerDeserializerFixture, AllChunkTypesWithStalenan) { @@ -472,44 +618,43 @@ TEST_F(SerializerDeserializerFixture, AllChunkTypesWithStalenan) { encoder_.encode(8, 134, STALE_NAN); // Act - serializer_.serialize(stream_); - Deserializer deserializer(get_buffer()); + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); // Assert - ASSERT_TRUE(deserializer.is_valid()); - ASSERT_EQ(10U, deserializer.get_chunks().size()); - EXPECT_TRUE(std::ranges::all_of(deserializer.get_chunks(), [](const auto& chunk) { return chunk.encoding_state.has_last_stalenan; })); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, deserializer.get_chunks()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, deserializer.get_chunks()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, deserializer.get_chunks()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, deserializer.get_chunks()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, deserializer.get_chunks()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, deserializer.get_chunks()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, deserializer.get_chunks()[9].label_set_id); + ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); + EXPECT_TRUE(std::ranges::all_of(serialized_view.get_chunks_view(), [](const auto& chunk) { return chunk.encoding_state.has_last_stalenan; })); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); + ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 100, .value = 1.0}, {.timestamp = 101, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 0))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 102, .value = 1.1}, {.timestamp = 103, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[1])))); + decode_current_chunk(serialized_view, 1))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 104, .value = 1.1}, {.timestamp = 105, .value = 1.2}, {.timestamp = 106, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[2])))); + decode_current_chunk(serialized_view, 2))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 107, .value = 1.0}, @@ -517,7 +662,7 @@ TEST_F(SerializerDeserializerFixture, AllChunkTypesWithStalenan) { {.timestamp = 109, .value = 3.0}, {.timestamp = 110, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[3])))); + decode_current_chunk(serialized_view, 3))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 111, .value = 1.1}, @@ -525,7 +670,7 @@ TEST_F(SerializerDeserializerFixture, AllChunkTypesWithStalenan) { {.timestamp = 113, .value = 3.1}, {.timestamp = 114, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[4])))); + decode_current_chunk(serialized_view, 4))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 115, .value = 1.1}, @@ -533,20 +678,20 @@ TEST_F(SerializerDeserializerFixture, AllChunkTypesWithStalenan) { {.timestamp = 117, .value = 3.1}, {.timestamp = 118, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[5])))); + decode_current_chunk(serialized_view, 5))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 119, .value = 2.0}, {.timestamp = 120, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[6])))); + decode_current_chunk(serialized_view, 6))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 121, .value = -1.0}, {.timestamp = 122, .value = -1.0}, {.timestamp = 123, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[7])))); + decode_current_chunk(serialized_view, 7))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 130, .value = 1.0}, @@ -555,14 +700,14 @@ TEST_F(SerializerDeserializerFixture, AllChunkTypesWithStalenan) { {.timestamp = 133, .value = 4.1}, {.timestamp = 134, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[8])))); + decode_current_chunk(serialized_view, 8))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 111, .value = 1.1}, {.timestamp = 112, .value = 2.1}, {.timestamp = 113, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[9])))); + decode_current_chunk(serialized_view, 20))); } TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypesWithStalenan) { @@ -619,44 +764,43 @@ TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypesWithStalenan) { ChunkFinalizer::finalize(storage_, 8, storage_.open_chunks[8]); // Act - serializer_.serialize(stream_); - Deserializer deserializer(get_buffer()); + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); // Assert - ASSERT_TRUE(deserializer.is_valid()); - ASSERT_EQ(10U, deserializer.get_chunks().size()); - EXPECT_TRUE(std::ranges::all_of(deserializer.get_chunks(), [](const auto& chunk) { return chunk.encoding_state.has_last_stalenan; })); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[0].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kDoubleConstant, deserializer.get_chunks()[1].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[2].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscInteger, deserializer.get_chunks()[3].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kValuesGorilla, deserializer.get_chunks()[4].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kGorilla, deserializer.get_chunks()[5].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kUint32Constant, deserializer.get_chunks()[6].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kFloat32Constant, deserializer.get_chunks()[7].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, deserializer.get_chunks()[8].encoding_state.encoding_type); - ASSERT_EQ(EncodingType::kTwoDoubleConstant, deserializer.get_chunks()[9].encoding_state.encoding_type); - ASSERT_EQ(20U, deserializer.get_chunks()[9].label_set_id); + ASSERT_EQ(10U, serialized_view.get_chunks_view().size()); + EXPECT_TRUE(std::ranges::all_of(serialized_view.get_chunks_view(), [](const auto& chunk) { return chunk.encoding_state.has_last_stalenan; })); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[0].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kDoubleConstant, serialized_view.get_chunks_view()[1].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[2].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscInteger, serialized_view.get_chunks_view()[3].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kValuesGorilla, serialized_view.get_chunks_view()[4].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kGorilla, serialized_view.get_chunks_view()[5].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kUint32Constant, serialized_view.get_chunks_view()[6].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kFloat32Constant, serialized_view.get_chunks_view()[7].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kAscIntegerThenValuesGorilla, serialized_view.get_chunks_view()[8].encoding_state.encoding_type); + ASSERT_EQ(EncodingType::kTwoDoubleConstant, serialized_view.get_chunks_view()[9].encoding_state.encoding_type); + ASSERT_EQ(20U, serialized_view.get_chunks_view()[9].label_set_id); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 100, .value = 1.0}, {.timestamp = 101, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[0])))); + decode_current_chunk(serialized_view, 0))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 102, .value = 1.1}, {.timestamp = 103, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[1])))); + decode_current_chunk(serialized_view, 1))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 104, .value = 1.1}, {.timestamp = 105, .value = 1.2}, {.timestamp = 106, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[2])))); + decode_current_chunk(serialized_view, 2))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 107, .value = 1.0}, @@ -664,7 +808,7 @@ TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypesWithStalenan) { {.timestamp = 109, .value = 3.0}, {.timestamp = 110, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[3])))); + decode_current_chunk(serialized_view, 3))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 111, .value = 1.1}, @@ -672,7 +816,7 @@ TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypesWithStalenan) { {.timestamp = 113, .value = 3.1}, {.timestamp = 114, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[4])))); + decode_current_chunk(serialized_view, 4))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 115, .value = 1.1}, @@ -680,20 +824,20 @@ TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypesWithStalenan) { {.timestamp = 117, .value = 3.1}, {.timestamp = 118, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[5])))); + decode_current_chunk(serialized_view, 5))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 119, .value = 2.0}, {.timestamp = 120, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[6])))); + decode_current_chunk(serialized_view, 6))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 121, .value = -1.0}, {.timestamp = 122, .value = -1.0}, {.timestamp = 123, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[7])))); + decode_current_chunk(serialized_view, 7))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 130, .value = 1.0}, @@ -702,64 +846,209 @@ TEST_F(SerializerDeserializerFixture, FinalizedAllChunkTypesWithStalenan) { {.timestamp = 133, .value = 4.1}, {.timestamp = 134, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[8])))); + decode_current_chunk(serialized_view, 8))); EXPECT_TRUE(std::ranges::equal( SampleList{ {.timestamp = 111, .value = 1.1}, {.timestamp = 112, .value = 2.1}, {.timestamp = 113, .value = STALE_NAN}, }, - decode_chunk(deserializer.create_decode_iterator(deserializer.get_chunks()[9])))); + decode_current_chunk(serialized_view, 20))); } -class DeserializerIteratorFixture : public SerializerDeserializerTrait, public testing::Test { +class SerializedDataNextIterFixture : public SerializerDeserializerTrait, public testing::Test { protected: - using DecodedChunks = std::vector; - - DecodedChunks decode_chunks() const { - DecodedChunks result; - for (auto& chunk : Deserializer{get_buffer()}) { - result.emplace_back(decode_chunk(Deserializer::create_decode_iterator(chunk))); + static std::vector get_chunks_ids(SerializedDataView& view) { + std::vector ans{}; + uint32_t id = view.next_series().first; + while (id != SerializedDataView::kNoMoreSeries) { + ans.push_back(id); + id = view.next_series().first; } - return result; + return ans; } }; -TEST_F(DeserializerIteratorFixture, EmptyChunksList) { +TEST_F(SerializedDataNextIterFixture, EmptyChunksList) { // Arrange // Act - serializer_.serialize({}, stream_); - auto decoded_chunks = decode_chunks(); + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + const auto ids = get_chunks_ids(serialized_view); + + // Assert + EXPECT_TRUE(ids.empty()); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); +} + +TEST_F(SerializedDataNextIterFixture, OneChunk) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 1.0); + + // Act + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + auto ids = get_chunks_ids(serialized_view); // Assert - EXPECT_TRUE(std::ranges::equal(DecodedChunks{}, decoded_chunks)); + EXPECT_TRUE(std::ranges::equal(ids, std::initializer_list{0u})); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); } -TEST_F(DeserializerIteratorFixture, OneChunk) { +TEST_F(SerializedDataNextIterFixture, OneChunkFinalized) { // Arrange encoder_.encode(0, 1, 1.0); encoder_.encode(0, 2, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 1.0); + encoder_.encode(0, 4, 1.0); // Act - serializer_.serialize({QueriedChunk{0}}, stream_); - auto decoded_chunks = decode_chunks(); + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + auto ids = get_chunks_ids(serialized_view); // Assert - EXPECT_TRUE(std::ranges::equal(DecodedChunks{SampleList{{.timestamp = 1, .value = 1.0}, {.timestamp = 2, .value = 1.0}}}, decoded_chunks)); + EXPECT_TRUE(std::ranges::equal(ids, std::initializer_list{0u})); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); } -TEST_F(DeserializerIteratorFixture, TwoChunks) { +TEST_F(SerializedDataNextIterFixture, SeveralChunks) { // Arrange encoder_.encode(0, 1, 1.0); + encoder_.encode(1, 1, 1.0); + + encoder_.encode(0, 2, 1.0); encoder_.encode(1, 2, 1.0); + encoder_.encode(0, 3, 1.0); + encoder_.encode(1, 3, 1.0); + + encoder_.encode(2, 1, 2.0); + encoder_.encode(2, 2, 2.0); + encoder_.encode(2, 3, 2.0); + + encoder_.encode(100, 4, 2.1); + encoder_.encode(100, 5, 2.2); + encoder_.encode(100, 7, 2.3); + + // Act + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + auto ids = get_chunks_ids(serialized_view); + + // Assert + EXPECT_TRUE(std::ranges::equal(ids, std::initializer_list{0u, 1u, 2u, 100u})); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); +} + +class SerializedDataIterFixture : public SerializerDeserializerTrait, public testing::Test {}; + +TEST_F(SerializedDataIterFixture, ResetIteratorToSameSeries) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(0, 2, 1.0); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + encoder_.encode(0, 3, 1.0); + encoder_.encode(0, 4, 1.0); + + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + auto [series_id, chunk_id] = serialized_view.next_series(); + + // Act + auto iter = serialized_view.create_series_iterator(chunk_id); + iter.reset(serialized_view.get_buffer_view(), serialized_view.get_chunks_view(), chunk_id); + + // Assert + EXPECT_EQ(series_id, 0u); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); + EXPECT_TRUE(std::ranges::equal(std::ranges::subrange(iter, DecodeIteratorSentinel{}), + std::initializer_list{Sample{.timestamp = 1, .value = 1.0}, Sample{.timestamp = 2, .value = 1.0}, + Sample{.timestamp = 3, .value = 1.0}, Sample{.timestamp = 4, .value = 1.0}})); +} + +TEST_F(SerializedDataIterFixture, ResetIteratorToAnotherSeries) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(1, 1, 1.0); + + encoder_.encode(0, 2, 1.0); + encoder_.encode(1, 2, 1.1); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); + + encoder_.encode(0, 3, 1.0); + encoder_.encode(1, 3, 1.2); + + encoder_.encode(0, 4, 1.0); + encoder_.encode(1, 4, 1.3); + + const auto serialized = serialize(); + SerializedDataView serialized_view(serialized); + + auto [series_id0, chunk_id0] = serialized_view.next_series(); + auto [series_id1, chunk_id1] = serialized_view.next_series(); + + // Act + auto iter = serialized_view.create_series_iterator(chunk_id0); + iter.reset(serialized_view.get_buffer_view(), serialized_view.get_chunks_view(), chunk_id1); + + // Assert + EXPECT_EQ(series_id0, 0u); + EXPECT_EQ(series_id1, 1u); + EXPECT_EQ(SerializedDataView::kNoMoreSeries, serialized_view.next_series().first); + EXPECT_TRUE(std::ranges::equal(std::ranges::subrange(iter, DecodeIteratorSentinel{}), + std::initializer_list{Sample{.timestamp = 1, .value = 1.0}, Sample{.timestamp = 2, .value = 1.1}, + Sample{.timestamp = 3, .value = 1.2}, Sample{.timestamp = 4, .value = 1.3}})); +} + +TEST_F(SerializedDataIterFixture, ResetIteratorToAnotherSerializedData) { + // Arrange + encoder_.encode(0, 1, 1.0); + encoder_.encode(1, 1, 1.0); + + encoder_.encode(0, 2, 1.0); + encoder_.encode(1, 2, 1.1); + ChunkFinalizer::finalize(storage_, 0, storage_.open_chunks[0]); + ChunkFinalizer::finalize(storage_, 1, storage_.open_chunks[1]); + + encoder_.encode(0, 3, 1.0); + encoder_.encode(1, 3, 1.2); + + encoder_.encode(0, 4, 1.0); + encoder_.encode(1, 4, 1.3); + + const auto serialized0 = serializer_.serialize(); + SerializedDataView serialized_view0(serialized0); + + encoder_.encode(0, 5, 1.0); + encoder_.encode(1, 5, 1.4); + + const auto serialized1 = serialize(); + SerializedDataView serialized_view1(serialized1); + + auto [series_id0, chunk_id0] = serialized_view0.next_series(); + + std::ignore = serialized_view1.next_series(); + auto [series_id1, chunk_id1] = serialized_view1.next_series(); + // Act - serializer_.serialize({QueriedChunk{0}, QueriedChunk{1}}, stream_); - auto decoded_chunks = decode_chunks(); + auto iter = serialized_view0.create_series_iterator(chunk_id0); + iter.reset(serialized_view1.get_buffer_view(), serialized_view1.get_chunks_view(), chunk_id1); // Assert - EXPECT_TRUE(std::ranges::equal(DecodedChunks{SampleList{{.timestamp = 1, .value = 1.0}}, SampleList{{.timestamp = 2, .value = 1.0}}}, decoded_chunks)); + EXPECT_TRUE( + std::ranges::equal(std::ranges::subrange(iter, DecodeIteratorSentinel{}), + std::initializer_list{Sample{.timestamp = 1, .value = 1.0}, Sample{.timestamp = 2, .value = 1.1}, Sample{.timestamp = 3, .value = 1.2}, + Sample{.timestamp = 4, .value = 1.3}, Sample{.timestamp = 5, .value = 1.4}})); } } // namespace \ No newline at end of file diff --git a/pp/series_index/reverse_index.h b/pp/series_index/reverse_index.h index c2c7e173d6..26b496bc35 100644 --- a/pp/series_index/reverse_index.h +++ b/pp/series_index/reverse_index.h @@ -69,7 +69,7 @@ class SeriesIdSequenceSnapshot { using difference_type = std::ptrdiff_t; Iterator(const SharedMemory::SharedPtr& memory, const DeltaRLE::Encoder& encoder) - : iterator_(DeltaRLE::DataSequence::decode_iterator(memory.get(), memory.constructed_item_count()), DeltaRLE::DataSequence::end(), &encoder), + : iterator_(DeltaRLE::DataSequence::decode_iterator(memory.get(), memory.items_count()), DeltaRLE::DataSequence::end(), &encoder), count_(encoder.count()) {} PROMPP_ALWAYS_INLINE Iterator& operator++() noexcept {