mikhail/llama.cpp-mtp-turboquant
1
0
Fork 0
Code Issues Pull Requests Actions 34 Packages Projects Releases Wiki Activity

common : gracefully handle incomplete output (#20191)

Browse Source 
* common : handle incomplete UTF-8 at end of input in PEG parser

* cont : if reached end prematurely, emit needs_more_input to propagate partial output

* cont: refactor peg parse context to add lenient flag

* cont : remove partial flag, keep lenient flag
This commit is contained in:
Aldehir Rojas
2026-03-08 11:17:02 -05:00
committed by GitHub
parent 9b24886f78
commit 451ef08432
11 changed files with 139 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/peg-parser/test-basic.cpp
+27 -27
View File
@@ -120,7 +120,7 @@ void test_basic(testing & t) {
return p.literal("hello") + p.optional(p.literal(" world"));
});
auto ctx = common_peg_parse_context("hello", false);
auto ctx = common_peg_parse_context("hello");
auto result = parser.parse(ctx);
t.assert_equal("optional_absent", true, result.success());
t.assert_equal("optional_absent_end", 5u, result.end);
@@ -132,7 +132,7 @@ void test_basic(testing & t) {
return p.literal("hello") + p.optional(p.literal(" world"));
});
auto ctx = common_peg_parse_context("hello ", true);
auto ctx = common_peg_parse_context("hello ", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("partial_match_need_more", true, result.need_more_input());
});
@@ -215,7 +215,7 @@ void test_basic(testing & t) {
t.test("sequence_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<thi", true);
auto ctx = common_peg_parse_context("<thi", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_partial_match_1", true, result.need_more_input());
});
@@ -224,7 +224,7 @@ void test_basic(testing & t) {
t.test("sequence_partial_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("begin") + p.literal("end"); });
auto ctx = common_peg_parse_context("begin", true);
auto ctx = common_peg_parse_context("begin", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_partial_match_2", true, result.need_more_input());
});
@@ -233,7 +233,7 @@ void test_basic(testing & t) {
t.test("sequence_partial_match_3", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<think></", true);
auto ctx = common_peg_parse_context("<think></", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_partial_match_3", true, result.need_more_input());
});
@@ -242,7 +242,7 @@ void test_basic(testing & t) {
t.test("sequence_full_match", [&](testing & t) {
auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal("world"); });
auto ctx = common_peg_parse_context("helloworld", false);
auto ctx = common_peg_parse_context("helloworld");
auto result = common_chat_combinator_parser.parse(ctx);
t.assert_equal("sequence_full_match", true, result.success());
});
@@ -251,7 +251,7 @@ void test_basic(testing & t) {
t.test("sequence_no_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });
auto ctx = common_peg_parse_context("<think>I am common_chat_combinator_parser", true);
auto ctx = common_peg_parse_context("<think>I am common_chat_combinator_parser", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("sequence_no_match", true, result.fail());
});
@@ -260,7 +260,7 @@ void test_basic(testing & t) {
t.test("choices_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); });
auto ctx = common_peg_parse_context("opt", true);
auto ctx = common_peg_parse_context("opt", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("choices_partial_match_1", true, result.need_more_input());
});
@@ -270,7 +270,7 @@ void test_basic(testing & t) {
auto parser =
build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); });
auto ctx = common_peg_parse_context("choice", true);
auto ctx = common_peg_parse_context("choice", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("choices_partial_match_2", true, result.need_more_input());
});
@@ -279,7 +279,7 @@ void test_basic(testing & t) {
t.test("choices_full_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); });
auto ctx = common_peg_parse_context("first", false);
auto ctx = common_peg_parse_context("first");
auto result = parser.parse(ctx);
t.assert_equal("choices_full_match_1", true, result.success());
});
@@ -288,7 +288,7 @@ void test_basic(testing & t) {
t.test("choices_full_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); });
auto ctx = common_peg_parse_context("beta", false);
auto ctx = common_peg_parse_context("beta");
auto result = parser.parse(ctx);
t.assert_equal("choices_full_match_2", true, result.success());
});
@@ -297,7 +297,7 @@ void test_basic(testing & t) {
t.test("choices_no_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); });
auto ctx = common_peg_parse_context("best", false);
auto ctx = common_peg_parse_context("best");
auto result = parser.parse(ctx);
t.assert_equal("choices_no_match", true, result.fail());
});
@@ -306,7 +306,7 @@ void test_basic(testing & t) {
t.test("zero_or_more_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); });
auto ctx = common_peg_parse_context("a", true);
auto ctx = common_peg_parse_context("a", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input());
});
@@ -315,7 +315,7 @@ void test_basic(testing & t) {
t.test("zero_or_more_partial_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); });
auto ctx = common_peg_parse_context("xyx", true);
auto ctx = common_peg_parse_context("xyx", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input());
});
@@ -324,7 +324,7 @@ void test_basic(testing & t) {
t.test("zero_or_more_full_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); });
auto ctx = common_peg_parse_context("test", false);
auto ctx = common_peg_parse_context("test");
auto result = parser.parse(ctx);
t.assert_equal("zero_or_more_full_match", true, result.success());
});
@@ -333,7 +333,7 @@ void test_basic(testing & t) {
t.test("one_or_more_partial_match_1", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); });
auto ctx = common_peg_parse_context("rep", true);
auto ctx = common_peg_parse_context("rep", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input());
});
@@ -342,7 +342,7 @@ void test_basic(testing & t) {
t.test("one_or_more_partial_match_2", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); });
auto ctx = common_peg_parse_context("aba", true);
auto ctx = common_peg_parse_context("aba", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input());
});
@@ -351,7 +351,7 @@ void test_basic(testing & t) {
t.test("one_or_more_full_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); });
auto ctx = common_peg_parse_context("single", false);
auto ctx = common_peg_parse_context("single");
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_full_match", true, result.success());
});
@@ -360,7 +360,7 @@ void test_basic(testing & t) {
t.test("one_or_more_no_match", [&](testing & t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); });
auto ctx = common_peg_parse_context("success", false);
auto ctx = common_peg_parse_context("success");
auto result = parser.parse(ctx);
t.assert_equal("one_or_more_no_match", true, result.fail());
});
@@ -376,7 +376,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("1", false);
common_peg_parse_context ctx("1");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@@ -390,7 +390,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[1]", false);
common_peg_parse_context ctx("[1]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@@ -404,7 +404,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[[2]]", false);
common_peg_parse_context ctx("[[2]]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@@ -418,7 +418,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[[[3]]]", false);
common_peg_parse_context ctx("[[[3]]]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_success", true, result.success());
@@ -432,7 +432,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[[", true);
common_peg_parse_context ctx("[[", COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
@@ -446,7 +446,7 @@ void test_basic(testing & t) {
return p.rule("value", p.ref("number") | p.ref("list"));
});
common_peg_parse_context ctx("[a]", false);
common_peg_parse_context ctx("[a]");
auto result = value_parser.parse(ctx);
t.assert_equal("result_is_fail", true, result.fail());
@@ -458,8 +458,8 @@ void test_basic(testing & t) {
return p.marker();
});
common_peg_parse_context ctx_square("[marker]", false);
common_peg_parse_context ctx_sharp("<marker>", false);
common_peg_parse_context ctx_square("[marker]");
common_peg_parse_context ctx_sharp("<marker>");
auto result_square = bracket_parser.parse(ctx_square);
auto result_sharp = bracket_parser.parse(ctx_sharp);
tests/peg-parser/test-json-parser.cpp
+6 -6
View File
@@ -46,7 +46,7 @@ void test_json_parser(testing &t) {
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
std::string input = R"({"name": "test", "value": )";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = json.parse(ctx);
@@ -58,7 +58,7 @@ void test_json_parser(testing &t) {
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
std::string input = R"([1, 2, 3, )";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = json.parse(ctx);
@@ -70,7 +70,7 @@ void test_json_parser(testing &t) {
auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });
std::string input = R"({"data": {"nested": )";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = json.parse(ctx);
@@ -84,7 +84,7 @@ void test_json_parser(testing &t) {
t.test("success", [&](testing &t) {
std::string input = R"("name": "bob")";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@@ -92,7 +92,7 @@ void test_json_parser(testing &t) {
t.test("partial", [&](testing &t) {
std::string input = R"("name": "bo)";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_true("need more input", result.need_more_input());
@@ -100,7 +100,7 @@ void test_json_parser(testing &t) {
t.test("failed", [&](testing &t) {
std::string input = R"([])";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("fail", result.fail());
tests/peg-parser/test-python-dict-parser.cpp
+3 -3
View File
@@ -85,7 +85,7 @@ void test_python_dict_parser(testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test', 'value': ";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
@@ -97,7 +97,7 @@ void test_python_dict_parser(testing &t) {
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
std::string input = "{'name': 'test";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
@@ -229,7 +229,7 @@ void test_python_dict_parser(testing &t) {
t.test("incomplete string", [&](testing &t) {
std::string input = "'hello";
common_peg_parse_context ctx(input, true);
common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
t.assert_true("need_more_input", result.need_more_input());
tests/peg-parser/test-unicode.cpp
+11 -14
View File
@@ -58,7 +58,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@@ -101,7 +101,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@@ -142,7 +142,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@@ -187,7 +187,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
// Assert result type matches
@@ -225,7 +225,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@@ -259,7 +259,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@@ -293,7 +293,7 @@ void test_unicode(testing &t) {
std::string test_name = "case " + std::to_string(i) + ": " + hex_dump(tc.input);
t.test(test_name, [&](testing &t) {
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@@ -330,7 +330,7 @@ void test_unicode(testing &t) {
return p.sequence({p.json_string_content(), p.literal("\"")});
});
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@@ -367,7 +367,7 @@ void test_unicode(testing &t) {
return p.json_string_content();
});
common_peg_parse_context ctx(tc.input, true);
common_peg_parse_context ctx(tc.input, COMMON_PEG_PARSE_FLAG_LENIENT);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@@ -390,9 +390,6 @@ void test_unicode(testing &t) {
// Invalid continuation byte
{std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL},
// Overlong encoding (security issue)
{std::string("\xC0\x80"), "", COMMON_PEG_PARSE_RESULT_FAIL},
};
for (size_t i = 0; i < test_cases.size(); i++) {
@@ -404,7 +401,7 @@ void test_unicode(testing &t) {
return p.json_string_content();
});
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
@@ -433,7 +430,7 @@ void test_unicode(testing &t) {
return p.sequence({p.json_string_content(), p.literal("\"")});
});
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
assert_result_equal(t, tc.expected_result, result.type);
tests/test-chat-auto-parser.cpp
+4 -4
View File
@@ -1478,7 +1478,7 @@ static void test_standard_json_tools_openai(testing & t) {
R"({"id": "call_abc123", "function": {"name": "get_current_weather", "arguments": {"location": "NYC"}}})"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
@@ -1524,7 +1524,7 @@ static void test_standard_json_tools_cohere(testing & t) {
R"({"tool_call_id": 0, "tool_name": "get_current_weather", "parameters": {"location": "NYC", "unit": "celsius"}})"
"]<|END_ACTION|>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
@@ -1570,7 +1570,7 @@ static void test_standard_json_tools_function_key(testing & t) {
R"({"get_current_weather": {"id": "call-0001", "args": {"location": "NYC", "unit": "celsius"}}})"
"]</tool_calls>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
@@ -1845,7 +1845,7 @@ static void test_tagged_args_with_embedded_quotes(testing & t) {
"</function>\n"
"</seed:tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
if (!t.assert_true("parse success", result.success())) {
tests/test-chat-peg-parser.cpp
+9 -9
View File
@@ -361,7 +361,7 @@ static void test_example_native(testing & t) {
t.log(line);
}
common_peg_parse_context ctx(tc.input, false);
common_peg_parse_context ctx(tc.input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@@ -458,7 +458,7 @@ static void test_example_qwen3_coder(testing & t) {
for (auto it = tokens.begin(); it != tokens.end(); it++) {
std::string in = std::accumulate(tokens.begin(), it + 1, std::string());
common_peg_parse_context ctx(in, it + 1 < tokens.end());
common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = parser.parse(ctx);
if (!t.assert_equal("not fail", false, result.fail())) {
@@ -523,7 +523,7 @@ static void test_example_qwen3_non_coder(testing & t) {
"\"fahrenheit\"}}"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@@ -556,7 +556,7 @@ static void test_example_qwen3_non_coder(testing & t) {
for (auto it = tokens.begin(); it != tokens.end(); it++) {
std::string in = std::accumulate(tokens.begin(), it + 1, std::string());
common_peg_parse_context ctx(in, it + 1 < tokens.end());
common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = parser.parse(ctx);
if (!t.assert_equal("not fail", false, result.fail())) {
@@ -617,7 +617,7 @@ void test_command7_parser_compare(testing & t) {
auto test_current = [&](const common_peg_arena & p, const std::string & input, bool is_partial,
bool print_results) {
common_peg_parse_context ctx(input, is_partial);
common_peg_parse_context ctx(input, is_partial ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = p.parse(ctx);
common_chat_msg msg;
@@ -780,7 +780,7 @@ static void test_prefix_tool_names(testing & t) {
"</function>"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@@ -814,7 +814,7 @@ static void test_prefix_tool_names(testing & t) {
for (auto it = tokens.begin(); it != tokens.end(); it++) {
std::string in = std::accumulate(tokens.begin(), it + 1, std::string());
common_peg_parse_context ctx(in, it + 1 < tokens.end());
common_peg_parse_context ctx(in, (it + 1 < tokens.end()) ? COMMON_PEG_PARSE_FLAG_LENIENT : COMMON_PEG_PARSE_FLAG_NONE);
auto result = parser.parse(ctx);
if (!t.assert_equal("not fail", false, result.fail())) {
@@ -864,7 +864,7 @@ static void test_prefix_tool_names(testing & t) {
"</function>"
"</tool_call>";
common_peg_parse_context ctx(input, false);
common_peg_parse_context ctx(input);
auto result = parser.parse(ctx);
t.assert_true("success", result.success());
@@ -931,7 +931,7 @@ static void test_tagged_peg_parser(testing & t) {
return p.tag("prefix", p.until(":")) + ":" + p.tag("value", p.rest()) + p.end();
});
auto result = parser.parse_and_extract("key:val", true);
auto result = parser.parse_and_extract("key:val", COMMON_PEG_PARSE_FLAG_LENIENT);
t.assert_true("not fail", !result.result.fail());
t.assert_equal("prefix tag", "key", result.tags.at("prefix"));
t.assert_equal("value tag", "val", result.tags.at("value"));