feat(modem): Add enhanced URC observer API

Author: david-cermak

components/esp_modem/examples/modem_console/main/modem_console_main.cpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Unlicense OR CC0-1.0
  */
@@ -97,10 +97,21 @@ void wakeup_modem(void)
 }
 
 #ifdef CONFIG_ESP_MODEM_URC_HANDLER
-command_result handle_urc(uint8_t *data, size_t len)
+esp_modem::DTE::UrcConsumeInfo handle_enhanced_urc(const esp_modem::DTE::UrcBufferInfo &info)
 {
-    ESP_LOG_BUFFER_HEXDUMP("on_read", data, len, ESP_LOG_INFO);
-    return command_result::TIMEOUT;
+    // Log buffer information for debugging
+    ESP_LOGI(TAG, "URC Buffer Info: total_size=%zu, processed_offset=%zu, new_data_size=%zu, command_active=%s",
+             info.buffer_total_size, info.processed_offset, info.new_data_size,
+             info.is_command_active ? "true" : "false");
+
+    // Log the new data content
+    if (info.new_data_size > 0) {
+        ESP_LOG_BUFFER_HEXDUMP("on_read", info.new_data_start, info.new_data_size, ESP_LOG_INFO);
+    }
+
+    // For console example, we just log and don't consume anything
+    // This allows the data to be processed by command handlers
+    return {esp_modem::DTE::UrcConsumeResult::CONSUME_NONE, 0};
 }
 #endif
 
@@ -381,14 +392,14 @@ extern "C" void app_main(void)
         CHECK_ERR(dce->reset(), ESP_LOGI(TAG, "OK"));
     });
 #ifdef CONFIG_ESP_MODEM_URC_HANDLER
-    const ConsoleCommand HandleURC("urc", "toggle urc handling", no_args, [&](ConsoleCommand * c) {
+    const ConsoleCommand HandleURC("urc", "toggle enhanced urc handling", no_args, [&](ConsoleCommand * c) {
         static int cnt = 0;
         if (++cnt % 2) {
-            ESP_LOGI(TAG, "Adding URC handler");
-            dce->set_urc(handle_urc);
+            ESP_LOGI(TAG, "Adding enhanced URC handler");
+            dce->set_enhanced_urc(handle_enhanced_urc);
         } else {
-            ESP_LOGI(TAG, "URC removed");
-            dce->set_urc(nullptr);
+            ESP_LOGI(TAG, "Enhanced URC removed");
+            dce->set_enhanced_urc(nullptr);
         }
         return 0;
     });

components/esp_modem/include/cxx_include/esp_modem_dce_template.hpp

@@ -103,6 +103,11 @@ class DCE_T {
     {
         dte->set_urc_cb(on_read_cb);
     }
+
+    void set_enhanced_urc(esp_modem::DTE::enhanced_urc_cb enhanced_cb)
+    {
+        dte->set_enhanced_urc_cb(enhanced_cb);
+    }
 #endif
 
     /**

components/esp_modem/include/cxx_include/esp_modem_dte.hpp

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -115,6 +115,42 @@ class DTE : public CommandableIf {
     {
         command_cb.urc_handler = std::move(line_cb);
     }
+
+    /**
+     * @brief Enhanced URC handler with buffer consumption control
+     * @param buffer_info Information about the current buffer state
+     * @return Information about how much of the buffer to consume
+     */
+    struct UrcBufferInfo {
+        const uint8_t* buffer_start;        // Start of entire buffer
+        size_t buffer_total_size;           // Total buffer size
+        size_t processed_offset;            // Offset of already processed data
+        size_t new_data_size;               // Size of new data since last call
+        const uint8_t* new_data_start;      // Pointer to start of new data
+        bool is_command_active;             // Whether a command is currently waiting for response
+    };
+
+    enum class UrcConsumeResult {
+        CONSUME_NONE,           // Don't consume anything, continue waiting
+        CONSUME_PARTIAL,        // Consume only part of the buffer
+        CONSUME_ALL             // Consume entire buffer
+    };
+
+    struct UrcConsumeInfo {
+        UrcConsumeResult result;
+        size_t consume_size;    // How many bytes to consume (0 = none, SIZE_MAX = all)
+    };
+
+    typedef std::function<UrcConsumeInfo(const UrcBufferInfo &)> enhanced_urc_cb;
+
+    /**
+     * @brief Set enhanced URC callback with buffer consumption control
+     * @param enhanced_cb Enhanced callback that can control buffer consumption
+     */
+    void set_enhanced_urc_cb(enhanced_urc_cb enhanced_cb)
+    {
+        command_cb.enhanced_urc_handler = std::move(enhanced_cb);
+    }
 #endif
 
     /**
@@ -171,6 +207,33 @@ class DTE : public CommandableIf {
     [[nodiscard]] bool exit_cmux();                         /*!< Exit of CMUX mode and cleanup  */
     void exit_cmux_internal();                              /*!< Cleanup CMUX */
 
+#ifdef CONFIG_ESP_MODEM_URC_HANDLER
+    /**
+     * @brief Buffer state tracking for enhanced URC processing
+     */
+    struct BufferState {
+        size_t total_processed = 0;         /*!< Total bytes processed by URC handlers */
+        size_t last_urc_processed = 0;      /*!< Last offset processed by URC */
+        bool command_waiting = false;       /*!< Whether command is waiting for response */
+        size_t command_start_offset = 0;    /*!< Where current command response started */
+    } buffer_state;
+
+    /**
+     * @brief Update buffer state when new data arrives
+     * @param new_data_size Size of new data added to buffer
+     */
+    void update_buffer_state(size_t new_data_size);
+
+    /**
+     * @brief Create URC buffer information for enhanced handlers
+     * @param data Buffer data pointer
+     * @param consumed Already consumed bytes
+     * @param len New data length
+     * @return UrcBufferInfo structure with complete buffer context
+     */
+    UrcBufferInfo create_urc_info(uint8_t* data, size_t consumed, size_t len);
+#endif
+
     Lock internal_lock{};                                   /*!< Locks DTE operations */
     unique_buffer buffer;                                   /*!< DTE buffer */
     std::shared_ptr<CMux> cmux_term;                        /*!< Primary terminal for this DTE */
@@ -216,14 +279,15 @@ class DTE : public CommandableIf {
     struct command_cb {
 #ifdef CONFIG_ESP_MODEM_URC_HANDLER
         got_line_cb urc_handler {};                             /*!< URC callback if enabled */
+        enhanced_urc_cb enhanced_urc_handler {};                /*!< Enhanced URC callback with consumption control */
 #endif
         static const size_t GOT_LINE = SignalGroup::bit0;       /*!< Bit indicating response available */
         got_line_cb got_line;                                   /*!< Supplied command callback */
         Lock line_lock{};                                       /*!< Command callback locking mechanism */
         char separator{};                                       /*!< Command reply separator (end of line/processing unit) */
         command_result result{};                                /*!< Command return code */
         SignalGroup signal;                                     /*!< Event group used to signal request-response operations */
-        bool process_line(uint8_t *data, size_t consumed, size_t len);  /*!< Lets the processing callback handle one line (processing unit) */
+        bool process_line(uint8_t *data, size_t consumed, size_t len, DTE* dte = nullptr);  /*!< Lets the processing callback handle one line (processing unit) */
         bool wait_for_line(uint32_t time_ms)                    /*!< Waiting for command processing */
         {
             return signal.wait_any(command_cb::GOT_LINE, time_ms);

components/esp_modem/src/esp_modem_dte.cpp

@@ -65,6 +65,10 @@ void DTE::set_command_callbacks()
 {
     primary_term->set_read_cb([this](uint8_t *data, size_t len) {
         Scoped<Lock> l(command_cb.line_lock);
+#ifdef CONFIG_ESP_MODEM_URC_HANDLER
+        // Update buffer state when new data arrives
+        update_buffer_state(len);
+#endif
 #ifndef CONFIG_ESP_MODEM_URC_HANDLER
         if (command_cb.got_line == nullptr || command_cb.result != command_result::TIMEOUT) {
             return false;   // this line has been processed already (got OK or FAIL previously)
@@ -80,7 +84,7 @@ void DTE::set_command_callbacks()
                 std::memcpy(inflatable.current(), data, len);
                 data = inflatable.begin();
             }
-            if (command_cb.process_line(data, inflatable.consumed, len)) {
+            if (command_cb.process_line(data, inflatable.consumed, len, this)) {
                 return true;
             }
             // at this point we're sure that the data processing hasn't finished,
@@ -92,7 +96,7 @@ void DTE::set_command_callbacks()
             inflatable.consumed += len;
             return false;
 #else
-            if (command_cb.process_line(data, 0, len)) {
+            if (command_cb.process_line(data, 0, len, this)) {
                 return true;
             }
             // cannot inflate and the processing hasn't finishes in the first iteration, but continue
@@ -105,7 +109,7 @@ void DTE::set_command_callbacks()
         if (buffer.size > buffer.consumed) {
             data = buffer.get();
             len = primary_term->read(data + buffer.consumed, buffer.size - buffer.consumed);
-            if (command_cb.process_line(data, buffer.consumed, len)) {
+            if (command_cb.process_line(data, buffer.consumed, len, this)) {
                 return true;
             }
             buffer.consumed += len;
@@ -121,7 +125,7 @@ void DTE::set_command_callbacks()
             inflatable.grow(inflatable.consumed + len);
         }
         len = primary_term->read(inflatable.current(), len);
-        if (command_cb.process_line(inflatable.begin(), inflatable.consumed, len)) {
+        if (command_cb.process_line(inflatable.begin(), inflatable.consumed, len, this)) {
             return true;
         }
         inflatable.consumed += len;
@@ -150,10 +154,19 @@ void DTE::set_command_callbacks()
 command_result DTE::command(const std::string &command, got_line_cb got_line, uint32_t time_ms, const char separator)
 {
     Scoped<Lock> l1(internal_lock);
+#ifdef CONFIG_ESP_MODEM_URC_HANDLER
+    // Track command start
+    buffer_state.command_waiting = true;
+    buffer_state.command_start_offset = buffer_state.total_processed;
+#endif
     command_cb.set(got_line, separator);
     primary_term->write((uint8_t *)command.c_str(), command.length());
     command_cb.wait_for_line(time_ms);
     command_cb.set(nullptr);
+#ifdef CONFIG_ESP_MODEM_URC_HANDLER
+    // Track command end
+    buffer_state.command_waiting = false;
+#endif
     buffer.consumed = 0;
 #ifdef CONFIG_ESP_MODEM_USE_INFLATABLE_BUFFER_IF_NEEDED
     inflatable.deflate();
@@ -365,18 +378,54 @@ void DTE::on_read(got_line_cb on_read_cb)
     });
 }
 
-bool DTE::command_cb::process_line(uint8_t *data, size_t consumed, size_t len)
+bool DTE::command_cb::process_line(uint8_t *data, size_t consumed, size_t len, DTE* dte)
 {
     // returning true indicates that the processing finished and lower layers can destroy the accumulated buffer
 #ifdef CONFIG_ESP_MODEM_URC_HANDLER
-    bool consume_buffer = false;
+    // Call enhanced URC handler if registered
+    if (enhanced_urc_handler && dte) {
+        // Create buffer info for enhanced URC handler
+        UrcBufferInfo buffer_info = dte->create_urc_info(data, consumed, len);
+
+        // Call enhanced URC handler
+        UrcConsumeInfo consume_info = enhanced_urc_handler(buffer_info);
+
+        // Handle consumption control
+        switch (consume_info.result) {
+        case UrcConsumeResult::CONSUME_NONE:
+            // Don't consume anything, continue with command processing
+            break;
+
+        case UrcConsumeResult::CONSUME_PARTIAL:
+            // Consume only specified amount
+            dte->buffer_state.last_urc_processed += consume_info.consume_size;
+            // Adjust data pointers for command processing
+            data += consume_info.consume_size;
+            consumed = (consumed + len) - consume_info.consume_size;
+            len = 0;
+            break;
+
+        case UrcConsumeResult::CONSUME_ALL:
+            // Consume entire buffer
+            dte->buffer_state.last_urc_processed = consumed + len;
+            return true;  // Signal buffer consumption
+        }
+    }
+
+    // Fallback to legacy URC handler if enhanced handler not set
     if (urc_handler) {
-        consume_buffer = urc_handler(data, consumed + len) != command_result::TIMEOUT;
+        bool consume_buffer = urc_handler(data, consumed + len) != command_result::TIMEOUT;
+        if (result != command_result::TIMEOUT || got_line == nullptr) {
+            return consume_buffer;   // this line has been processed already (got OK or FAIL previously)
+        }
     }
+#endif
+
+    // Continue with normal command processing
     if (result != command_result::TIMEOUT || got_line == nullptr) {
-        return consume_buffer;   // this line has been processed already (got OK or FAIL previously)
+        return false;  // Command processing continues
     }
-#endif
+
     if (memchr(data + consumed, separator, len)) {
         result = got_line(data, consumed + len);
         if (result == command_result::OK || result == command_result::FAIL) {
@@ -423,3 +472,22 @@ void DTE::extra_buffer::grow(size_t need_size)
  */
 unique_buffer::unique_buffer(size_t size):
     data(std::make_unique<uint8_t[]>(size)), size(size), consumed(0) {}
+
+#ifdef CONFIG_ESP_MODEM_URC_HANDLER
+void DTE::update_buffer_state(size_t new_data_size)
+{
+    buffer_state.total_processed += new_data_size;
+}
+
+DTE::UrcBufferInfo DTE::create_urc_info(uint8_t* data, size_t consumed, size_t len)
+{
+    return {
+        .buffer_start = data,
+        .buffer_total_size = consumed + len,
+        .processed_offset = buffer_state.last_urc_processed,
+        .new_data_size = (consumed + len) - buffer_state.last_urc_processed,
+        .new_data_start = data + buffer_state.last_urc_processed,
+        .is_command_active = buffer_state.command_waiting
+    };
+}
+#endif