diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 4d564af9e2..3b7de22a24 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -328,9 +328,7 @@ uint16_t APIConnection::encode_message_to_buffer(ProtoMessage &msg, uint8_t mess #endif // Calculate size - ProtoSize size_calc; - msg.calculate_size(size_calc); - uint32_t calculated_size = size_calc.get_size(); + uint32_t calculated_size = msg.calculated_size(); // Cache frame sizes to avoid repeated virtual calls const uint8_t header_padding = conn->helper_->frame_header_padding(); @@ -358,19 +356,13 @@ uint16_t APIConnection::encode_message_to_buffer(ProtoMessage &msg, uint8_t mess shared_buf.resize(current_size + footer_size + header_padding); } - // Encode directly into buffer - size_t size_before_encode = shared_buf.size(); - msg.encode({&shared_buf}); + // Pre-resize buffer to include payload, then encode through raw pointer + size_t write_start = shared_buf.size(); + shared_buf.resize(write_start + calculated_size); + msg.encode(ProtoWriteBuffer{&shared_buf, write_start}); - // Calculate actual encoded size (not including header that was already added) - size_t actual_payload_size = shared_buf.size() - size_before_encode; - - // Return actual total size (header + actual payload + footer) - size_t actual_total_size = header_padding + actual_payload_size + footer_size; - - // Verify that calculate_size() returned the correct value - assert(calculated_size == actual_payload_size); - return static_cast(actual_total_size); + // Return total size (header + payload + footer) + return static_cast(header_padding + calculated_size + footer_size); } #ifdef USE_BINARY_SENSOR @@ -1827,12 +1819,13 @@ bool APIConnection::try_to_clear_buffer(bool log_out_of_space) { return false; } bool APIConnection::send_message_impl(const ProtoMessage &msg, uint8_t message_type) { - ProtoSize size; - msg.calculate_size(size); + uint32_t payload_size = msg.calculated_size(); std::vector &shared_buf = this->parent_->get_shared_buffer_ref(); - this->prepare_first_message_buffer(shared_buf, size.get_size()); - msg.encode({&shared_buf}); - return this->send_buffer({&shared_buf}, message_type); + this->prepare_first_message_buffer(shared_buf, payload_size); + size_t write_start = shared_buf.size(); + shared_buf.resize(write_start + payload_size); + msg.encode(ProtoWriteBuffer{&shared_buf, write_start}); + return this->send_buffer(ProtoWriteBuffer{&shared_buf}, message_type); } bool APIConnection::send_buffer(ProtoWriteBuffer buffer, uint8_t message_type) { const bool is_log_message = (message_type == SubscribeLogsResponse::MESSAGE_TYPE); diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index 743f51dac7..b780bbe82e 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -111,7 +111,7 @@ void DeviceInfoResponse::encode(ProtoWriteBuffer buffer) const { } #endif #ifdef USE_AREAS - buffer.encode_message(22, this->area); + buffer.encode_message(22, this->area, false); #endif #ifdef USE_ZWAVE_PROXY buffer.encode_uint32(23, this->zwave_proxy_feature_flags); @@ -2435,7 +2435,7 @@ void VoiceAssistantRequest::encode(ProtoWriteBuffer buffer) const { buffer.encode_bool(1, this->start); buffer.encode_string(2, this->conversation_id); buffer.encode_uint32(3, this->flags); - buffer.encode_message(4, this->audio_settings); + buffer.encode_message(4, this->audio_settings, false); buffer.encode_string(5, this->wake_word_phrase); } void VoiceAssistantRequest::calculate_size(ProtoSize &size) const { diff --git a/esphome/components/api/proto.cpp b/esphome/components/api/proto.cpp index 764dd3f391..9db7e91282 100644 --- a/esphome/components/api/proto.cpp +++ b/esphome/components/api/proto.cpp @@ -70,6 +70,16 @@ uint32_t ProtoDecodableMessage::count_repeated_field(const uint8_t *buffer, size return count; } +#ifdef ESPHOME_DEBUG_API +void ProtoWriteBuffer::debug_check_bounds_(size_t bytes, const char *caller) { + if (this->pos_ + bytes > this->buffer_->data() + this->buffer_->size()) { + ESP_LOGE(TAG, "ProtoWriteBuffer bounds check failed in %s: bytes=%zu offset=%td buf_size=%zu", caller, bytes, + this->pos_ - this->buffer_->data(), this->buffer_->size()); + abort(); + } +} +#endif + void ProtoDecodableMessage::decode(const uint8_t *buffer, size_t length) { const uint8_t *ptr = buffer; const uint8_t *end = buffer + length; diff --git a/esphome/components/api/proto.h b/esphome/components/api/proto.h index 8ac79633cf..db07b0df44 100644 --- a/esphome/components/api/proto.h +++ b/esphome/components/api/proto.h @@ -217,21 +217,26 @@ class Proto32Bit { class ProtoWriteBuffer { public: - ProtoWriteBuffer(std::vector *buffer) : buffer_(buffer) {} - void write(uint8_t value) { this->buffer_->push_back(value); } + ProtoWriteBuffer(std::vector *buffer) : buffer_(buffer), pos_(buffer->data() + buffer->size()) {} + ProtoWriteBuffer(std::vector *buffer, size_t write_pos) + : buffer_(buffer), pos_(buffer->data() + write_pos) {} void encode_varint_raw(uint32_t value) { while (value > 0x7F) { - this->buffer_->push_back(static_cast(value | 0x80)); + this->debug_check_bounds_(1); + *this->pos_++ = static_cast(value | 0x80); value >>= 7; } - this->buffer_->push_back(static_cast(value)); + this->debug_check_bounds_(1); + *this->pos_++ = static_cast(value); } void encode_varint_raw_64(uint64_t value) { while (value > 0x7F) { - this->buffer_->push_back(static_cast(value | 0x80)); + this->debug_check_bounds_(1); + *this->pos_++ = static_cast(value | 0x80); value >>= 7; } - this->buffer_->push_back(static_cast(value)); + this->debug_check_bounds_(1); + *this->pos_++ = static_cast(value); } /** * Encode a field key (tag/wire type combination). @@ -245,23 +250,18 @@ class ProtoWriteBuffer { * * Following https://protobuf.dev/programming-guides/encoding/#structure */ - void encode_field_raw(uint32_t field_id, uint32_t type) { - uint32_t val = (field_id << 3) | (type & WIRE_TYPE_MASK); - this->encode_varint_raw(val); - } + void encode_field_raw(uint32_t field_id, uint32_t type) { this->encode_varint_raw((field_id << 3) | type); } void encode_string(uint32_t field_id, const char *string, size_t len, bool force = false) { if (len == 0 && !force) return; this->encode_field_raw(field_id, 2); // type 2: Length-delimited string this->encode_varint_raw(len); - - // Using resize + memcpy instead of insert provides significant performance improvement: - // ~10-11x faster for 16-32 byte strings, ~3x faster for 64-byte strings - // as it avoids iterator checks and potential element moves that insert performs - size_t old_size = this->buffer_->size(); - this->buffer_->resize(old_size + len); - std::memcpy(this->buffer_->data() + old_size, string, len); + // Direct memcpy into pre-sized buffer — avoids push_back() per-byte capacity checks + // and vector::insert() iterator overhead. ~10-11x faster for 16-32 byte strings. + this->debug_check_bounds_(len); + std::memcpy(this->pos_, string, len); + this->pos_ += len; } void encode_string(uint32_t field_id, const std::string &value, bool force = false) { this->encode_string(field_id, value.data(), value.size(), force); @@ -288,17 +288,25 @@ class ProtoWriteBuffer { if (!value && !force) return; this->encode_field_raw(field_id, 0); // type 0: Varint - bool - this->buffer_->push_back(value ? 0x01 : 0x00); + this->debug_check_bounds_(1); + *this->pos_++ = value ? 0x01 : 0x00; } - void encode_fixed32(uint32_t field_id, uint32_t value, bool force = false) { + __attribute__((noinline)) void encode_fixed32(uint32_t field_id, uint32_t value, bool force = false) { if (value == 0 && !force) return; this->encode_field_raw(field_id, 5); // type 5: 32-bit fixed32 - this->write((value >> 0) & 0xFF); - this->write((value >> 8) & 0xFF); - this->write((value >> 16) & 0xFF); - this->write((value >> 24) & 0xFF); + this->debug_check_bounds_(4); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + // Protobuf fixed32 is little-endian, so direct copy works + std::memcpy(this->pos_, &value, 4); + this->pos_ += 4; +#else + *this->pos_++ = (value >> 0) & 0xFF; + *this->pos_++ = (value >> 8) & 0xFF; + *this->pos_++ = (value >> 16) & 0xFF; + *this->pos_++ = (value >> 24) & 0xFF; +#endif } // NOTE: Wire type 1 (64-bit fixed: double, fixed64, sfixed64) is intentionally // not supported to reduce overhead on embedded systems. All ESPHome devices are @@ -334,11 +342,19 @@ class ProtoWriteBuffer { } /// Encode a packed repeated sint32 field (zero-copy from vector) void encode_packed_sint32(uint32_t field_id, const std::vector &values); - void encode_message(uint32_t field_id, const ProtoMessage &value); + /// Encode a nested message field (force=true for repeated, false for singular) + void encode_message(uint32_t field_id, const ProtoMessage &value, bool force = true); std::vector *get_buffer() const { return buffer_; } protected: +#ifdef ESPHOME_DEBUG_API + void debug_check_bounds_(size_t bytes, const char *caller = __builtin_FUNCTION()); +#else + void debug_check_bounds_(size_t bytes, const char *caller = __builtin_FUNCTION()) {} +#endif + std::vector *buffer_; + uint8_t *pos_; }; #ifdef HAS_PROTO_MESSAGE_DUMP @@ -419,6 +435,8 @@ class ProtoMessage { virtual void encode(ProtoWriteBuffer buffer) const {} // Default implementation for messages with no fields virtual void calculate_size(ProtoSize &size) const {} + // Convenience: calculate and return size directly (defined after ProtoSize) + uint32_t calculated_size() const; #ifdef HAS_PROTO_MESSAGE_DUMP virtual const char *dump_to(DumpBuffer &out) const = 0; virtual const char *message_name() const { return "unknown"; } @@ -877,6 +895,14 @@ class ProtoSize { } }; +// Implementation of methods that depend on ProtoSize being fully defined + +inline uint32_t ProtoMessage::calculated_size() const { + ProtoSize size; + this->calculate_size(size); + return size.get_size(); +} + // Implementation of encode_packed_sint32 - must be after ProtoSize is defined inline void ProtoWriteBuffer::encode_packed_sint32(uint32_t field_id, const std::vector &values) { if (values.empty()) @@ -897,31 +923,28 @@ inline void ProtoWriteBuffer::encode_packed_sint32(uint32_t field_id, const std: } // Implementation of encode_message - must be after ProtoMessage is defined -inline void ProtoWriteBuffer::encode_message(uint32_t field_id, const ProtoMessage &value) { - this->encode_field_raw(field_id, 2); // type 2: Length-delimited message - +inline void ProtoWriteBuffer::encode_message(uint32_t field_id, const ProtoMessage &value, bool force) { // Calculate the message size first ProtoSize msg_size; value.calculate_size(msg_size); uint32_t msg_length_bytes = msg_size.get_size(); - // Calculate how many bytes the length varint needs - uint32_t varint_length_bytes = ProtoSize::varint(msg_length_bytes); + // Skip empty singular messages (matches add_message_field which skips when nested_size == 0) + // Repeated messages (force=true) are always encoded since an empty item is meaningful + if (msg_length_bytes == 0 && !force) + return; - // Reserve exact space for the length varint - size_t begin = this->buffer_->size(); - this->buffer_->resize(this->buffer_->size() + varint_length_bytes); + this->encode_field_raw(field_id, 2); // type 2: Length-delimited message - // Write the length varint directly - encode_varint_to_buffer(msg_length_bytes, this->buffer_->data() + begin); + // Write the length varint directly through pos_ + this->encode_varint_raw(msg_length_bytes); - // Now encode the message content - it will append to the buffer + // Encode nested message - value.encode() gets a copy of *this with current pos_. + // The copy writes msg_length_bytes bytes starting from our current pos_. + // We then advance our pos_ by the known message size. value.encode(*this); - -#ifdef ESPHOME_DEBUG_API - // Verify that the encoded size matches what we calculated - assert(this->buffer_->size() == begin + varint_length_bytes + msg_length_bytes); -#endif + this->debug_check_bounds_(msg_length_bytes); + this->pos_ += msg_length_bytes; } // Implementation of decode_to_message - must be after ProtoDecodableMessage is defined diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index 4fbee49dae..add2415c05 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -689,6 +689,14 @@ class MessageType(TypeInfo): def encode_func(self) -> str: return "encode_message" + @property + def encode_content(self) -> str: + # Singular message fields pass force=false (skip empty messages) + # The default for encode_nested_message is force=true (for repeated fields) + return ( + f"buffer.{self.encode_func}({self.number}, this->{self.field_name}, false);" + ) + @property def decode_length(self) -> str: # Override to return None for message types because we can't use template-based