Autoparser - complete refactoring of parser architecture (#18675)

* Autoparser - full single commit squish

* Final pre-merge changes: minor fixes, Kimi 2.5 model parser
This commit is contained in:
Piotr Wilkin (ilintar)
2026-03-06 21:01:00 +01:00
committed by GitHub
parent 34df42f7be
commit 566059a26b
63 changed files with 12967 additions and 10071 deletions
+7 -2
View File
@@ -2,6 +2,8 @@
#include <cstdint>
#include <string_view>
#include <vector>
#include <string>
// UTF-8 parsing utilities for streaming-aware unicode support
@@ -16,7 +18,10 @@ struct utf8_parse_result {
// Determine the expected length of a UTF-8 sequence from its first byte
// Returns 0 for invalid first bytes
size_t utf8_sequence_length(unsigned char first_byte);
size_t common_utf8_sequence_length(unsigned char first_byte);
// Parse a single UTF-8 codepoint from input
utf8_parse_result parse_utf8_codepoint(std::string_view input, size_t offset);
utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t offset);
std::string common_unicode_cpts_to_utf8(const std::vector<uint32_t> & cps);
std::string common_unicode_cpt_to_utf8(uint32_t cpt);