Autoparser - complete refactoring of parser architecture (#18675)
* Autoparser - full single commit squish * Final pre-merge changes: minor fixes, Kimi 2.5 model parser
This commit is contained in:
Piotr Wilkin (ilintar)
committed by
GitHub
GitHub
parent
34df42f7be
commit
566059a26b
@@ -187,11 +187,11 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
|
||||
# llama_build_and_test(test-double-float.cpp) # SLOW
|
||||
endif()
|
||||
|
||||
llama_build_and_test(test-chat-parser.cpp)
|
||||
llama_build_and_test(test-chat-peg-parser.cpp peg-parser/simple-tokenize.cpp)
|
||||
llama_build_and_test(test-chat-template.cpp)
|
||||
llama_build_and_test(test-jinja.cpp)
|
||||
llama_test(test-jinja NAME test-jinja-py ARGS -py LABEL python)
|
||||
llama_build_and_test(test-chat-auto-parser.cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
|
||||
llama_build_and_test(test-chat-template.cpp)
|
||||
llama_build_and_test(test-json-partial.cpp)
|
||||
llama_build_and_test(test-log.cpp)
|
||||
llama_build_and_test(
|
||||
@@ -201,6 +201,7 @@ llama_build_and_test(
|
||||
peg-parser/test-gbnf-generation.cpp
|
||||
peg-parser/test-json-parser.cpp
|
||||
peg-parser/test-json-serialization.cpp
|
||||
peg-parser/test-python-dict-parser.cpp
|
||||
peg-parser/test-unicode.cpp
|
||||
peg-parser/tests.h
|
||||
)
|
||||
@@ -279,3 +280,5 @@ target_link_libraries(${TEST_TARGET} PRIVATE llama)
|
||||
|
||||
llama_build_and_test(test-alloc.cpp)
|
||||
target_include_directories(test-alloc PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src)
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#include "peg-parser.h"
|
||||
#include "tests.h"
|
||||
|
||||
void test_basic(testing & t) {
|
||||
@@ -450,5 +451,21 @@ void test_basic(testing & t) {
|
||||
|
||||
t.assert_equal("result_is_fail", true, result.fail());
|
||||
});
|
||||
|
||||
// Test markers
|
||||
t.test("marker", [](testing &t) {
|
||||
auto bracket_parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.marker();
|
||||
});
|
||||
|
||||
common_peg_parse_context ctx_square("[marker]", false);
|
||||
common_peg_parse_context ctx_sharp("<marker>", false);
|
||||
|
||||
auto result_square = bracket_parser.parse(ctx_square);
|
||||
auto result_sharp = bracket_parser.parse(ctx_sharp);
|
||||
|
||||
t.assert_true("result_square_is_success", result_square.success());
|
||||
t.assert_true("result_sharp_is_success", result_sharp.success());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
#include "tests.h"
|
||||
|
||||
void test_python_dict_parser(testing &t) {
|
||||
// Test parsing a simple Python dict object with single quotes
|
||||
t.test("simple Python dict object parsing", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'name': 'test', 'value': 42, 'flag': True}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing a Python array with mixed types
|
||||
t.test("Python array with mixed types", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "[1, 'hello', True, None, 3.14]";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing nested Python dict with objects and arrays
|
||||
t.test("nested Python dict with objects and arrays", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input =
|
||||
"{'users': [{'id': 1, 'name': 'Alice'}, {'id': 2, 'name': 'Bob'}], 'count': 2, 'metadata': {'version': '1.0', 'tags': ['admin', 'user']}}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing Python dict with escaped single quotes
|
||||
t.test("Python dict with escaped single quotes", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'message': 'It\\'s working!'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test parsing Python dict with double quotes inside single quotes
|
||||
t.test("Python dict with double quotes inside single quotes", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'quote': 'He said \"Hello\"'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test the example from the requirements
|
||||
t.test("complex Python dict example from requirements", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete object
|
||||
t.test("need_more_input() parsing - incomplete object", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'name': 'test', 'value': ";
|
||||
common_peg_parse_context ctx(input, true);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test need_more_input() parsing - incomplete single-quoted string
|
||||
t.test("need_more_input() parsing - incomplete single-quoted string", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'name': 'test";
|
||||
common_peg_parse_context ctx(input, true);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_need_more_input", true, result.need_more_input());
|
||||
});
|
||||
|
||||
// Test unicode in Python dict strings
|
||||
t.test("unicode in Python dict strings", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'message': 'Hello, 世界!'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test Python dict with unicode escapes
|
||||
t.test("Python dict with unicode escapes", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{'unicode': 'Hello\\u0041'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test that Python parser accepts double-quoted strings too
|
||||
t.test("Python parser accepts double-quoted strings", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{\"name\": \"test\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test Python parser with mixed quote styles
|
||||
t.test("Python parser with mixed quote styles", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
std::string input = "{\"name\": 'test', 'value': \"hello\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
|
||||
t.assert_equal("result_is_success", true, result.success());
|
||||
t.assert_equal("result_end", input.size(), result.end);
|
||||
});
|
||||
|
||||
// Test Python True/False/None
|
||||
t.test("Python True/False/None", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.python_value(); });
|
||||
|
||||
t.test("True", [&](testing &t) {
|
||||
std::string input = "True";
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("False", [&](testing &t) {
|
||||
std::string input = "False";
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("None", [&](testing &t) {
|
||||
std::string input = "None";
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("rejects JSON-style true/false/null", [&](testing &t) {
|
||||
for (const auto & kw : {"true", "false", "null"}) {
|
||||
std::string input = kw;
|
||||
common_peg_parse_context ctx(input);
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true(std::string("rejects ") + kw, result.fail());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Test single-quoted string content parser directly
|
||||
t.test("single-quoted string content parser", [](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.sequence({ p.literal("'"), p.single_quoted_string_content(), p.literal("'"), p.space() });
|
||||
});
|
||||
|
||||
t.test("simple string", [&](testing &t) {
|
||||
std::string input = "'hello'";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("string with escaped single quote", [&](testing &t) {
|
||||
std::string input = "'it\\'s'";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("string with double quotes", [&](testing &t) {
|
||||
std::string input = "'say \"hello\"'";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("incomplete string", [&](testing &t) {
|
||||
std::string input = "'hello";
|
||||
common_peg_parse_context ctx(input, true);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("need_more_input", result.need_more_input());
|
||||
});
|
||||
});
|
||||
|
||||
// Test json() with pre-registered flexible json-string rule (python dict support)
|
||||
t.test("json() parser with flexible json-string rule", [](testing &t) {
|
||||
t.test("json() rejects single quotes by default", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{'name': 'test'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("fail", result.fail());
|
||||
});
|
||||
|
||||
t.test("json() accepts single quotes with pre-registered flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
// Pre-register json-string rule with both quote styles
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{'name': 'test'}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("json() still accepts double quotes with flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{\"name\": \"test\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("json() accepts mixed quote styles with flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{\"name\": 'test', 'value': \"hello\"}";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
|
||||
t.test("complex nested structure with flexible json-string rule", [&](testing &t) {
|
||||
auto parser = build_peg_parser([](common_peg_parser_builder & p) {
|
||||
p.rule("json-string", [&]() {
|
||||
return p.choice({ p.double_quoted_string(), p.single_quoted_string() });
|
||||
});
|
||||
return p.json();
|
||||
});
|
||||
|
||||
std::string input = "{ 'obj' : { 'something': 1, 'other \"something\"' : 'foo\\'s bar' } }";
|
||||
common_peg_parse_context ctx(input);
|
||||
|
||||
auto result = parser.parse(ctx);
|
||||
t.assert_true("success", result.success());
|
||||
t.assert_equal("end", input.size(), result.end);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -22,3 +22,4 @@ void test_json_parser(testing &t);
|
||||
void test_gbnf_generation(testing &t);
|
||||
void test_unicode(testing &t);
|
||||
void test_json_serialization(testing &t);
|
||||
void test_python_dict_parser(testing &t);
|
||||
|
||||
@@ -7820,6 +7820,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1, 64, 256, {1, 1}, {1, 1}));
|
||||
}
|
||||
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 6, 4096, 5120, {1, 1}, {1, 1}));
|
||||
|
||||
#if 0
|
||||
// test the mat-mat path for Metal
|
||||
for (int k = 1; k < 512; ++k) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,617 +0,0 @@
|
||||
// Tests chat handling, including grammar generation and parsing for tool calling, for various templates.
|
||||
//
|
||||
// Also acts as a CLI to generate a Markdown summary of the formats of Jinja templates,
|
||||
// e.g. given Minja (http://github.com/google/minja) checked out in parent dir:
|
||||
//
|
||||
// cmake -B build && cmake --build build --parallel && ./build/bin/test-chat ../minja/build/tests/*.jinja 2>/dev/null
|
||||
//
|
||||
#include <exception>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
#include "chat-parser.h"
|
||||
#include "common.h"
|
||||
#include "log.h"
|
||||
#include "regex-partial.h"
|
||||
|
||||
template <class T>
|
||||
static void assert_equals(const std::string_view label, const T & expected, const T & actual) {
|
||||
if (expected != actual) {
|
||||
std::cerr << label << std::endl;
|
||||
std::cerr << "Expected: " << expected << std::endl;
|
||||
std::cerr << "Actual: " << actual << std::endl;
|
||||
std::cerr << std::flush;
|
||||
throw std::runtime_error("Test failed");
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
static void assert_equals(const T & expected, const T & actual) {
|
||||
assert_equals("", expected, actual);
|
||||
}
|
||||
static void assert_equals(const char * expected, const std::string & actual) {
|
||||
return assert_equals<std::string>(expected, actual);
|
||||
}
|
||||
|
||||
static void assert_throws(const std::function<void()> & fn, const std::string & expected_exception_pattern = "") {
|
||||
try {
|
||||
fn();
|
||||
} catch (const std::exception & e) {
|
||||
if (expected_exception_pattern.empty()) {
|
||||
return;
|
||||
}
|
||||
std::regex expected_exception_regex(expected_exception_pattern);
|
||||
std::string actual_message = e.what();
|
||||
if (std::regex_search(actual_message, expected_exception_regex)) {
|
||||
return;
|
||||
}
|
||||
throw std::runtime_error("Exception doesn't match expected pattern: " + actual_message + " (pattern: " + expected_exception_pattern + ")");
|
||||
throw std::runtime_error("Exception of unexpected type: " + std::string(e.what()));
|
||||
}
|
||||
throw std::runtime_error("Exception was expected but not thrown");
|
||||
}
|
||||
|
||||
static void test_reasoning() {
|
||||
//common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG);
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_NONE;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
common_chat_msg_parser builder("<tnk>Cogito</tnk>Ergo sum", /* is_partial= */ false, params);
|
||||
assert_equals(false, builder.try_parse_reasoning("<tnk>", "</tnk>"));
|
||||
assert_equals("<tnk>Cogito</tnk>Ergo sum", builder.consume_rest());
|
||||
}
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
common_chat_msg_parser builder("<tnk>Cogito</tnk>Ergo sum", /* is_partial= */ false, params);
|
||||
assert_equals(true, builder.try_parse_reasoning("<tnk>", "</tnk>"));
|
||||
assert_equals(std::string("Cogito"), builder.result().reasoning_content);
|
||||
assert_equals("Ergo sum", builder.consume_rest());
|
||||
}
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_NONE;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
common_chat_msg_parser builder("Cogito</tnk>Ergo sum", /* is_partial= */ false, params);
|
||||
assert_equals(false, builder.try_parse_reasoning("<tnk>", "</tnk>"));
|
||||
assert_equals("Cogito</tnk>Ergo sum", builder.consume_rest());
|
||||
}
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
common_chat_msg_parser builder("Cogito</tnk>Ergo sum", /* is_partial= */ false, params);
|
||||
assert_equals(true, builder.try_parse_reasoning("<tnk>", "</tnk>"));
|
||||
assert_equals(std::string("Cogito"), builder.result().reasoning_content);
|
||||
assert_equals("Ergo sum", builder.consume_rest());
|
||||
}
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = true;
|
||||
params.thinking_forced_open = true;
|
||||
common_chat_msg_parser builder("Cogito</tnk>Ergo sum", /* is_partial= */ false, params);
|
||||
assert_equals(true, builder.try_parse_reasoning("<tnk>", "</tnk>"));
|
||||
assert_equals("<think>Cogito</think>", builder.result().content);
|
||||
assert_equals("Ergo sum", builder.consume_rest());
|
||||
}
|
||||
{
|
||||
const std::string variant("content_only_inline_think");
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_CONTENT_ONLY;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
params.parse_tool_calls = false;
|
||||
const std::string input = "<think>Pense</think>Bonjour";
|
||||
auto msg = common_chat_parse(input, false, params);
|
||||
assert_equals(variant, std::string("Pense"), msg.reasoning_content);
|
||||
assert_equals(variant, std::string("Bonjour"), msg.content);
|
||||
}
|
||||
{
|
||||
const std::string variant("llama_3_inline_think");
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_LLAMA_3_X;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
params.parse_tool_calls = false;
|
||||
const std::string input = "<think>Plan</think>Réponse";
|
||||
auto msg = common_chat_parse(input, false, params);
|
||||
assert_equals(variant, std::string("Plan"), msg.reasoning_content);
|
||||
assert_equals(variant, std::string("Réponse"), msg.content);
|
||||
}
|
||||
// Test DeepSeek V3.1 parsing - reasoning content followed by "</think>" and then regular content
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("deepseek_v3_1_reasoning_format_deepseek");
|
||||
common_chat_msg_parser builder("REASONING</think>ok", /* is_partial= */ false, params);
|
||||
assert_equals(variant, true, builder.try_parse_reasoning("<think>", "</think>"));
|
||||
assert_equals(variant, std::string("REASONING"), builder.result().reasoning_content);
|
||||
assert_equals(variant, std::string("ok"), builder.consume_rest());
|
||||
}
|
||||
// Test DeepSeek V3.1 parsing - reasoning_format none - reasoning content followed by "</think>" and then regular content
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_NONE;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("deepseek_v3_1_reasoning_format_none");
|
||||
const std::string input = "REASONING</think>ok";
|
||||
auto msg = common_chat_parse(input, false, params);
|
||||
assert_equals(variant, std::string("REASONING</think>ok"), msg.content);
|
||||
assert_equals(variant, std::string(""), msg.reasoning_content);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_regex() {
|
||||
auto test_throws = [](const std::string & input, const std::string & regex, const std::string & expected_exception_pattern = "") {
|
||||
common_chat_msg_parser builder(input, /* is_partial= */ false, {});
|
||||
assert_throws([&]() { builder.consume_regex(common_regex(regex)); }, expected_exception_pattern);
|
||||
};
|
||||
|
||||
test_throws("Hello, world!", "abc", "^abc$");
|
||||
test_throws("Hello, world!", "e", "^e$");
|
||||
|
||||
{
|
||||
common_chat_msg_parser builder("Hello, world!", /* is_partial= */ false, {});
|
||||
builder.consume_regex(common_regex("Hello"));
|
||||
assert_equals(", world!", builder.consume_rest());
|
||||
}
|
||||
|
||||
{
|
||||
// When in non partial mode, we can say whether the regex was consumed or not.
|
||||
common_chat_msg_parser builder("Hello,", /* is_partial= */ false, {});
|
||||
assert_equals(false, builder.try_consume_regex(common_regex("Hello, world!")).has_value());
|
||||
}
|
||||
{
|
||||
common_chat_msg_parser builder("Hello,", /* is_partial= */ false, {});
|
||||
auto res = builder.try_consume_regex(common_regex("H(el)l(?:o, world!)?"));
|
||||
assert_equals(true, res.has_value());
|
||||
// Verify captures
|
||||
assert_equals<size_t>(2, res->groups.size());
|
||||
assert_equals("Hell", builder.str(res->groups[0]));
|
||||
assert_equals("el", builder.str(res->groups[1]));
|
||||
// Verify position is after the match
|
||||
assert_equals<size_t>(4, builder.pos());
|
||||
assert_equals("o,", builder.consume_rest());
|
||||
}
|
||||
{
|
||||
// But in partial mode, we have a partial final match / can't decide, so we throw a partial exception.
|
||||
common_chat_msg_parser builder("Hello,", /* is_partial= */ true, {});
|
||||
assert_throws([&]() {
|
||||
builder.try_consume_regex(common_regex("Hello, world!"));
|
||||
}, "^Hello, world!$");
|
||||
}
|
||||
|
||||
// Now regardless of the mode, we can tell these aren't a match.
|
||||
for (const auto is_partial : {false, true}) {
|
||||
common_chat_msg_parser builder("Hello,", is_partial, {});
|
||||
assert_equals(false, builder.try_consume_regex(common_regex("a(b|c)(d|e)f")).has_value());
|
||||
}
|
||||
for (const auto is_partial : {false, true}) {
|
||||
common_chat_msg_parser builder("Hello,", is_partial, {});
|
||||
assert_equals(false, builder.try_consume_literal("Oh"));
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<std::string> barely_healable_jsons = {
|
||||
"{",
|
||||
"{\"",
|
||||
"{\"\\",
|
||||
"{\"n",
|
||||
"{\"name\"",
|
||||
"{\"name\":",
|
||||
"{\"name\":\"",
|
||||
"{\"name\":\"\\",
|
||||
"{\"name\":\"python",
|
||||
"{\"name\":\"python\\",
|
||||
"{\",",
|
||||
"{\":",
|
||||
"{\"[",
|
||||
"{\"]",
|
||||
"{\"{",
|
||||
"{\"}",
|
||||
"{\"1",
|
||||
"{\"name\":\",",
|
||||
"{\"name\":\":",
|
||||
"{\"name\":\"[",
|
||||
"{\"name\":\"]",
|
||||
"{\"name\":\"{",
|
||||
"{\"name\":\"}",
|
||||
"{\"name\":\"1",
|
||||
};
|
||||
|
||||
static void test(const std::string & input, bool is_partial, const std::vector<std::vector<std::string>> & args_paths, const std::vector<std::vector<std::string>> & content_paths, const std::string & expected) {
|
||||
common_chat_msg_parser builder(input, is_partial, {});
|
||||
auto js = builder.try_consume_json_with_dumped_args(args_paths, content_paths);
|
||||
assert_equals(true, js.has_value());
|
||||
assert_equals(is_partial, js->is_partial);
|
||||
assert_equals(expected, args_paths.size() == 1 && args_paths[0].empty() ? js->value.get<std::string>() : js->value.dump());
|
||||
}
|
||||
|
||||
static void test_deepseek_v3_1_tool_calls() {
|
||||
//common_log_set_verbosity_thold(LOG_DEFAULT_DEBUG);
|
||||
// variant: happy path for when it works as the model card says it should
|
||||
const std::string variant("simple");
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string input = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>";
|
||||
auto msg = common_chat_parse(input, false, params);
|
||||
assert_equals<std::size_t>(variant, 1, msg.tool_calls.size());
|
||||
assert_equals(variant, std::string("get_time"), msg.tool_calls[0].name);
|
||||
// JSON arguments are dumped without spaces
|
||||
assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments);
|
||||
assert_equals(variant, std::string(""), msg.content);
|
||||
assert_equals(variant, std::string(""), msg.reasoning_content);
|
||||
|
||||
// variant: simple + thinking open
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("simple_thinking");
|
||||
const std::string in = "REASONING</think><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>";
|
||||
auto m = common_chat_parse(in, false, params);
|
||||
assert_equals<std::size_t>(variant, 1, m.tool_calls.size());
|
||||
assert_equals(variant, std::string("get_time"), m.tool_calls[0].name);
|
||||
assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments);
|
||||
assert_equals(variant, std::string(""), m.content);
|
||||
assert_equals(variant, std::string("REASONING"), m.reasoning_content);
|
||||
}
|
||||
// variant: simple + multiple tool calls
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("simple_multiple_tool_calls");
|
||||
const std::string in = "CONTENT<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>{\"city\": \"Paris\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>";
|
||||
auto m = common_chat_parse(in, false, params);
|
||||
assert_equals<std::size_t>(variant, 2, m.tool_calls.size());
|
||||
assert_equals(variant, std::string("get_time"), m.tool_calls[0].name);
|
||||
assert_equals(variant, std::string("{\"city\":\"Paris\"}"), m.tool_calls[0].arguments);
|
||||
assert_equals(variant, std::string("get_weather"), m.tool_calls[1].name);
|
||||
assert_equals(variant, std::string("{\"city\":\"Paris\"}"), m.tool_calls[1].arguments);
|
||||
assert_equals(variant, std::string("CONTENT"), m.content);
|
||||
assert_equals(variant, std::string(""), m.reasoning_content);
|
||||
}
|
||||
|
||||
|
||||
// variant: thinking forced open + tool call in reasoning content
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("thinking_forced_open_tool_call_in_reasoning");
|
||||
const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING</think><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>";
|
||||
auto m = common_chat_parse(in, false, params);
|
||||
assert_equals<std::size_t>(variant, 1, m.tool_calls.size());
|
||||
assert_equals(variant, std::string("get_time"), m.tool_calls[0].name);
|
||||
assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments);
|
||||
assert_equals(variant, std::string(""), m.content);
|
||||
assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time2<｜tool▁sep｜>{\"city\": \"Tokyo2\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>REASONING"), m.reasoning_content);
|
||||
}
|
||||
|
||||
// variant: thinking forced open + tool call in reasoning content + no closing think + not partial
|
||||
// This is a bit of a fine tuning issue on the model's part IMO. It really should not be attempting
|
||||
// to make tool calls in reasoning content according to the model card, but it does sometimes, so
|
||||
// add the reasoning content as regular content and parse the tool calls.
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("thinking_forced_open_tool_call_in_reasoning_no_closing_think_not_partial");
|
||||
const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>";
|
||||
auto m = common_chat_parse(in, false, params);
|
||||
assert_equals(variant, std::string("REASONING"), m.content);
|
||||
assert_equals(variant, std::string(""), m.reasoning_content);
|
||||
assert_equals<std::size_t>(variant, 1, m.tool_calls.size());
|
||||
assert_equals(variant, std::string("get_time"), m.tool_calls[0].name);
|
||||
assert_equals(variant, std::string("{\"city\":\"Tokyo\"}"), m.tool_calls[0].arguments);
|
||||
}
|
||||
|
||||
// variant: thinking forced open + tool call in reasoning content + no closing think + partial
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("thinking_forced_open_tool_call_in_reasoning_no_closing_think_partial");
|
||||
const std::string in = "REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>";
|
||||
auto m = common_chat_parse(in, /* is_partial= */ true, params);
|
||||
assert_equals(variant, std::string("REASONING<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_time<｜tool▁sep｜>{\"city\": \"Tokyo\"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>"), m.reasoning_content);
|
||||
assert_equals(variant, std::string(""), m.content);
|
||||
assert_equals<std::size_t>(variant, 0, m.tool_calls.size());
|
||||
}
|
||||
|
||||
// variant: thinking not forced open + reasoning + regular content + no tool calls
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = true;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("thinking_forced_open_reasoning_regular_content_no_tool_calls");
|
||||
const std::string in = "REASONING</think>CONTENT";
|
||||
auto m = common_chat_parse(in, false, params);
|
||||
assert_equals<std::size_t>(variant, 0, m.tool_calls.size());
|
||||
assert_equals(variant, std::string("CONTENT"), m.content);
|
||||
assert_equals(variant, std::string("REASONING"), m.reasoning_content);
|
||||
}
|
||||
// variant: thinking not forced open + missing reasoning + no tool calls
|
||||
{
|
||||
common_chat_parser_params params;
|
||||
params.format = COMMON_CHAT_FORMAT_DEEPSEEK_V3_1;
|
||||
params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
|
||||
params.reasoning_in_content = false;
|
||||
params.thinking_forced_open = false;
|
||||
params.parse_tool_calls = true;
|
||||
const std::string variant("thinking_not_forced_open_missing_reasoning_no_tool_calls");
|
||||
const std::string in = "CONTENT";
|
||||
auto m = common_chat_parse(in, false, params);
|
||||
assert_equals<std::size_t>(variant, 0, m.tool_calls.size());
|
||||
assert_equals(variant, std::string("CONTENT"), m.content);
|
||||
assert_equals(variant, std::string(""), m.reasoning_content);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_with_args(const std::string & input, const std::string & expected, bool parse_as_partial = true, bool is_partial = true) {
|
||||
common_chat_msg_parser builder(input, parse_as_partial, {});
|
||||
auto js = builder.try_consume_json_with_dumped_args({{"args"}}, {});
|
||||
assert_equals(true, js.has_value());
|
||||
assert_equals(is_partial, js->is_partial);
|
||||
assert_equals(expected, js->value.dump());
|
||||
}
|
||||
|
||||
static void test_json_with_dumped_args_no_args() {
|
||||
// Normal JSON, nothing to heal, nothing to dump
|
||||
test("{\"name\": \"python\"}", false, {}, {}, "{\"name\":\"python\"}");
|
||||
// Full json is args
|
||||
test("{\"name\": \"python\"}", false, {{}}, {}, "{\"name\":\"python\"}");
|
||||
|
||||
// If the arguments are further down, don't heal partial content.
|
||||
for (const auto & src : barely_healable_jsons) {
|
||||
test(src, true, {{"arguments"}}, {}, "{}");
|
||||
}
|
||||
// But heal content that isn't partial.
|
||||
test("{\"name\": \"python\"", true, {{"arguments"}}, {}, "{\"name\":\"python\"}");
|
||||
}
|
||||
|
||||
static void test_json_with_dumped_args() {
|
||||
|
||||
// Partial content.
|
||||
test("{\"content\": \"t", true, {}, {{"content"}}, "{\"content\":\"t\"}");
|
||||
test("{\"content\": \"", true, {}, {{"content"}}, "{\"content\":\"\"}");
|
||||
test("{\"content\": ", true, {}, {{"content"}}, "{}");
|
||||
|
||||
// If the entire JSON is the arguments, healing it them dumping it produces the same output as the input (just reformatted).
|
||||
test("{\"name\": \"python", true, {{}}, {}, "{\"name\":\"python");
|
||||
for (const auto & src : barely_healable_jsons) {
|
||||
test(src, true, {{}}, {}, src);
|
||||
}
|
||||
|
||||
// Full JSON w/ args
|
||||
for (auto parse_as_partial : {true, false}) {
|
||||
test_with_args(
|
||||
R"({"name": "python", "args": {"arg1": 1}})",
|
||||
R"({"name":"python","args":"{\"arg1\":1}"})",
|
||||
parse_as_partial,
|
||||
/* is_partial= */ false
|
||||
);
|
||||
}
|
||||
|
||||
// Partial JSON w/ partial args
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {")",
|
||||
R"({"foo":"bar","args":"{\""})"
|
||||
);
|
||||
// Partial args broken in object key
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"ar)",
|
||||
R"({"foo":"bar","args":"{\"ar"})"
|
||||
);
|
||||
// Partial args broken after object key
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1")",
|
||||
R"({"foo":"bar","args":"{\"arg1\""})"
|
||||
);
|
||||
// Partial args broken before object value
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1":)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":"})"
|
||||
);
|
||||
// Partial args broken before object value (space)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": )",
|
||||
R"({"foo":"bar","args":"{\"arg1\":"})"
|
||||
);
|
||||
// Partial args broken in object value that may not be complete (int)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": 1)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":"})"
|
||||
);
|
||||
// Partial args broken in object value that is complete (int)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": 1 )",
|
||||
R"({"foo":"bar","args":"{\"arg1\":1"})"
|
||||
);
|
||||
// Partial args broken in object value that is incomplete (string)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": ")",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\""})"
|
||||
);
|
||||
// Partial args broken in object value that is complete (string)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "1")",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"1\""})"
|
||||
);
|
||||
// Partial args broken on array opening
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": [)",
|
||||
R"({"foo":"bar","args":"["})"
|
||||
);
|
||||
// Partial args broken on array value that is incomplete (int)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": [1)",
|
||||
R"({"foo":"bar","args":"["})"
|
||||
);
|
||||
// Partial args broken on array value that is complete (int)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": [1 )",
|
||||
R"({"foo":"bar","args":"[1"})"
|
||||
);
|
||||
// Partial args broken on array value that is complete (string)
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": ["1")",
|
||||
R"({"foo":"bar","args":"[\"1\""})"
|
||||
);
|
||||
// Partial args broken after array value
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": [1,)",
|
||||
R"({"foo":"bar","args":"[1,"})"
|
||||
);
|
||||
// Partial args broken on nested array
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": [)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":["})"
|
||||
);
|
||||
|
||||
// Unicode tests
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\u)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\u"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\u0)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\u0"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\u00)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\u00"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\u000)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\u000"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\u0000)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\u0000"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud8)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud8"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud80)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud80"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud800)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud800"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud800\)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud800\\"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud800\u)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud800\\u"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud800\ud)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud800\\ud"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud800\udc)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud800\\udc"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud800\udc0)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud800\\udc0"})"
|
||||
);
|
||||
test_with_args(
|
||||
R"({"foo": "bar", "args": {"arg1": "\ud800\udc00)",
|
||||
R"({"foo":"bar","args":"{\"arg1\":\"\\ud800\\udc00"})"
|
||||
);
|
||||
}
|
||||
|
||||
static void test_positions() {
|
||||
{
|
||||
common_chat_msg_parser builder("Hello, world!", /* is_partial= */ false, {});
|
||||
assert_equals<size_t>(0, builder.pos());
|
||||
assert_throws([&]() { builder.move_to(100); });
|
||||
assert_equals<size_t>(0, builder.pos());
|
||||
assert_throws([&]() { builder.move_back(1); });
|
||||
assert_equals<size_t>(0, builder.pos());
|
||||
|
||||
builder.move_to(8);
|
||||
assert_equals<size_t>(8, builder.pos());
|
||||
builder.move_back(1);
|
||||
assert_equals<size_t>(7, builder.pos());
|
||||
assert_equals("world!", builder.consume_rest());
|
||||
|
||||
builder.move_to(0);
|
||||
assert_equals<size_t>(0, builder.pos());
|
||||
|
||||
assert_throws([&]() { builder.finish(); });
|
||||
assert_equals<size_t>(0, builder.pos());
|
||||
|
||||
builder.move_to(builder.input().size());
|
||||
builder.finish();
|
||||
}
|
||||
{
|
||||
common_chat_msg_parser builder("Hello, world!", /* is_partial= */ true, {});
|
||||
|
||||
builder.move_to(builder.input().size());
|
||||
assert_equals<size_t>(builder.input().size(), builder.pos());
|
||||
builder.finish();
|
||||
}
|
||||
}
|
||||
|
||||
int main() {
|
||||
test_positions();
|
||||
test_json_with_dumped_args_no_args();
|
||||
test_json_with_dumped_args();
|
||||
test_reasoning();
|
||||
test_regex();
|
||||
test_deepseek_v3_1_tool_calls();
|
||||
std::cout << "All tests passed!\n";
|
||||
return 0;
|
||||
}
|
||||
+675
-468
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,5 @@
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
#include <regex>
|
||||
@@ -21,17 +22,16 @@
|
||||
|
||||
using json = nlohmann::ordered_json;
|
||||
|
||||
int main_automated_tests(void);
|
||||
static int main_automated_tests(void);
|
||||
|
||||
void run_multiple(std::string dir_path, bool stop_on_first_failure, json input, bool use_common = false);
|
||||
void run_single(std::string contents, json input, bool use_common = false, const std::string & output_path = "");
|
||||
static void run_multiple(const std::string& dir_path, bool stop_on_first_failure, const json& input, bool use_common = false);
|
||||
static void run_single(const std::string& contents, json input, bool use_common = false, const std::string & output_path = "");
|
||||
|
||||
|
||||
|
||||
std::string HELP = R"(
|
||||
static std::string HELP = R"(
|
||||
Usage: test-chat-template [OPTIONS] PATH_TO_TEMPLATE
|
||||
Options:
|
||||
-h, --help Show this help message and exit.
|
||||
--with-tools Add a tool and a tool call to the default JSON input
|
||||
--json <path> Path to the JSON input file.
|
||||
--stop-on-first-fail Stop testing on the first failure (default: false).
|
||||
--no-common Use direct Jinja engine instead of common chat templates (default: use common).
|
||||
@@ -41,7 +41,7 @@ If PATH_TO_TEMPLATE is a directory, runs all .jinja files in that directory.
|
||||
If PATH_TO_TEMPLATE is omitted, runs automated tests (default CI mode).
|
||||
)";
|
||||
|
||||
std::string DEFAULT_JSON = R"({
|
||||
static std::string DEFAULT_JSON = R"({
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
@@ -57,12 +57,65 @@ std::string DEFAULT_JSON = R"({
|
||||
"add_generation_prompt": true
|
||||
})";
|
||||
|
||||
static std::string DEFAULT_JSON_WITH_TOOLS = R"({
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello, how are you?"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "I am fine, thank you!"
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Call a tool!"
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call00001",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "test",
|
||||
"arguments": { "arg": "hello" }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "test",
|
||||
"description": "Test",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"arg": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["arg"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"add_generation_prompt": true
|
||||
})";
|
||||
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
std::vector<std::string> args(argv, argv + argc);
|
||||
|
||||
std::string tmpl_path;
|
||||
std::string json_path;
|
||||
std::string output_path;
|
||||
std::string & json_to_use = DEFAULT_JSON;
|
||||
bool stop_on_first_fail = false;
|
||||
bool use_common = true;
|
||||
|
||||
@@ -70,9 +123,12 @@ int main(int argc, char ** argv) {
|
||||
if (args[i] == "--help" || args[i] == "-h") {
|
||||
std::cout << HELP << "\n";
|
||||
return 0;
|
||||
} else if (args[i] == "--json" && i + 1 < args.size()) {
|
||||
}
|
||||
if (args[i] == "--json" && i + 1 < args.size()) {
|
||||
json_path = args[i + 1];
|
||||
i++;
|
||||
} else if (args[i] == "--with-tools") {
|
||||
json_to_use = DEFAULT_JSON_WITH_TOOLS;
|
||||
} else if (args[i] == "--stop-on-first-fail") {
|
||||
stop_on_first_fail = true;
|
||||
} else if (args[i] == "--output" && i + 1 < args.size()) {
|
||||
@@ -105,7 +161,7 @@ int main(int argc, char ** argv) {
|
||||
std::istreambuf_iterator<char>());
|
||||
input_json = json::parse(content);
|
||||
} else {
|
||||
input_json = json::parse(DEFAULT_JSON);
|
||||
input_json = json::parse(json_to_use);
|
||||
}
|
||||
|
||||
std::filesystem::path p(tmpl_path);
|
||||
@@ -125,7 +181,7 @@ int main(int argc, char ** argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
void run_multiple(std::string dir_path, bool stop_on_first_fail, json input, bool use_common) {
|
||||
void run_multiple(const std::string& dir_path, bool stop_on_first_fail, const json& input, bool use_common) {
|
||||
std::vector<std::string> failed_tests;
|
||||
|
||||
// list all files in models/templates/ and run each
|
||||
@@ -180,7 +236,7 @@ static std::string format_using_common(
|
||||
common_chat_templates_inputs inputs;
|
||||
inputs.use_jinja = true;
|
||||
inputs.messages = messages;
|
||||
inputs.tools = tools;
|
||||
inputs.tools = std::move(tools);
|
||||
inputs.add_generation_prompt = true;
|
||||
auto output = common_chat_templates_apply(tmpls.get(), inputs).prompt;
|
||||
output = normalize_newlines(output);
|
||||
@@ -209,7 +265,7 @@ static jinja::value_string format_using_direct_engine(
|
||||
|
||||
jinja::runtime runtime(ctx);
|
||||
const jinja::value results = runtime.execute(ast);
|
||||
auto parts = runtime.gather_string_parts(results);
|
||||
auto parts = jinja::runtime::gather_string_parts(results);
|
||||
|
||||
std::cout << "\n=== RESULTS ===\n";
|
||||
for (const auto & part : parts->as_string().parts) {
|
||||
@@ -220,7 +276,7 @@ static jinja::value_string format_using_direct_engine(
|
||||
}
|
||||
|
||||
|
||||
void run_single(std::string contents, json input, bool use_common, const std::string & output_path) {
|
||||
void run_single(const std::string& contents, json input, bool use_common, const std::string & output_path) {
|
||||
jinja::enable_debug(true);
|
||||
|
||||
jinja::value_string output_parts;
|
||||
@@ -560,7 +616,7 @@ int main_automated_tests(void) {
|
||||
supported_tmpl.resize(res);
|
||||
res = llama_chat_builtin_templates(supported_tmpl.data(), supported_tmpl.size());
|
||||
std::cout << "Built-in chat templates:\n";
|
||||
for (auto tmpl : supported_tmpl) {
|
||||
for (const auto *tmpl : supported_tmpl) {
|
||||
std::cout << " " << tmpl << "\n";
|
||||
}
|
||||
|
||||
@@ -592,6 +648,7 @@ int main_automated_tests(void) {
|
||||
}
|
||||
|
||||
std::vector<common_chat_msg> messages;
|
||||
messages.reserve(conversation.size());
|
||||
for (const auto & msg : conversation) {
|
||||
messages.push_back(simple_msg(msg.role, msg.content));
|
||||
}
|
||||
@@ -622,58 +679,6 @@ int main_automated_tests(void) {
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: llama_chat_format_single will be deprecated, remove these tests later
|
||||
|
||||
// test llama_chat_format_single for system message
|
||||
std::cout << "\n\n=== llama_chat_format_single (system message) ===\n\n";
|
||||
std::vector<common_chat_msg> chat2;
|
||||
auto sys_msg = simple_msg("system", "You are a helpful assistant");
|
||||
|
||||
auto fmt_sys = [&](std::string tmpl_str) {
|
||||
auto tmpls = common_chat_templates_init(/* model= */ nullptr, tmpl_str);
|
||||
auto output = common_chat_format_single(tmpls.get(), chat2, sys_msg, false, /* use_jinja= */ false);
|
||||
std::cout << "fmt_sys(" << tmpl_str << ") : " << output << "\n";
|
||||
std::cout << "-------------------------\n";
|
||||
return output;
|
||||
};
|
||||
assert(fmt_sys("chatml") == "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n");
|
||||
assert(fmt_sys("mistral-v1") == " [INST] You are a helpful assistant\n\n");
|
||||
assert(fmt_sys("mistral-v3") == "[INST] You are a helpful assistant\n\n");
|
||||
assert(fmt_sys("mistral-v3-tekken") == "[INST]You are a helpful assistant\n\n");
|
||||
assert(fmt_sys("mistral-v7") == "[SYSTEM_PROMPT] You are a helpful assistant[/SYSTEM_PROMPT]");
|
||||
assert(fmt_sys("llama2") == "[INST] You are a helpful assistant\n");
|
||||
assert(fmt_sys("llama2-sys") == "[INST] <<SYS>>\nYou are a helpful assistant\n<</SYS>>\n\n");
|
||||
assert(fmt_sys("mistral") == "[INST] You are a helpful assistant\n"); // for old pre-v1 templates
|
||||
assert(fmt_sys("gemma") == ""); // for gemma, system message is merged with user message
|
||||
assert(fmt_sys("llama3") == "<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|>");
|
||||
assert(fmt_sys("gigachat") == "<s>You are a helpful assistant<|message_sep|>");
|
||||
|
||||
|
||||
// test llama_chat_format_single for user message
|
||||
std::cout << "\n\n=== llama_chat_format_single (user message) ===\n\n";
|
||||
chat2.push_back(simple_msg("system", "You are a helpful assistant"));
|
||||
chat2.push_back(simple_msg("user", "Hello"));
|
||||
chat2.push_back(simple_msg("assistant", "I am assistant"));
|
||||
auto new_msg = simple_msg("user", "How are you");
|
||||
|
||||
auto fmt_single = [&](const std::string & tmpl_str) {
|
||||
auto tmpls = common_chat_templates_init(/* model= */ nullptr, tmpl_str.c_str());
|
||||
auto output = common_chat_format_single(tmpls.get(), chat2, new_msg, true, /* use_jinja= */ false);
|
||||
std::cout << "fmt_single(" << tmpl_str << ") : " << output << "\n";
|
||||
std::cout << "-------------------------\n";
|
||||
return output;
|
||||
};
|
||||
assert(fmt_single("chatml") == "\n<|im_start|>user\nHow are you<|im_end|>\n<|im_start|>assistant\n");
|
||||
assert(fmt_single("mistral-v1") == " [INST] How are you [/INST]");
|
||||
assert(fmt_single("mistral-v3") == "[INST] How are you[/INST]");
|
||||
assert(fmt_single("mistral-v3-tekken") == "[INST]How are you[/INST]");
|
||||
assert(fmt_single("mistral-v7") == "[INST] How are you[/INST]");
|
||||
assert(fmt_single("llama2") == "[INST] How are you [/INST]");
|
||||
assert(fmt_single("mistral") == "[INST] How are you [/INST]"); // for old pre-v1 templates
|
||||
assert(fmt_single("gemma") == "\n<start_of_turn>user\nHow are you<end_of_turn>\n<start_of_turn>model\n");
|
||||
assert(fmt_single("llama3") == "<|start_header_id|>user<|end_header_id|>\n\nHow are you<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n");
|
||||
// assert(fmt_single("gigachat") == "user<|role_sep|>How are you<|message_sep|>available functions<|role_sep|>[]<|message_sep|>assistant<|role_sep|>");
|
||||
|
||||
std::cout << "\nOK: All tests passed successfully.\n";
|
||||
|
||||
return 0;
|
||||
|
||||
+2457
-3334
File diff suppressed because it is too large
Load Diff
@@ -1340,6 +1340,26 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
|
||||
)"""
|
||||
});
|
||||
|
||||
test({
|
||||
SUCCESS,
|
||||
"description only (no type) treated as unconstrained",
|
||||
R"""({"description": "The 0-based index of the last line to be retrieved (inclusive). If None, read until the end of the file."})""",
|
||||
R"""(
|
||||
array ::= "[" space ( value ("," space value)* )? "]" space
|
||||
boolean ::= ("true" | "false") space
|
||||
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
|
||||
decimal-part ::= [0-9]{1,16}
|
||||
integral-part ::= [0] | [1-9] [0-9]{0,15}
|
||||
null ::= "null" space
|
||||
number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
|
||||
object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space
|
||||
root ::= value
|
||||
space ::= | " " | "\n"{1,2} [ \t]{0,20}
|
||||
string ::= "\"" char* "\"" space
|
||||
value ::= object | array | string | number | boolean | null
|
||||
)"""
|
||||
});
|
||||
|
||||
test({
|
||||
SUCCESS,
|
||||
"literal string with escapes",
|
||||
|
||||
@@ -20,6 +20,7 @@ int main(int argc, char *argv[]) {
|
||||
t.test("json", test_json_parser);
|
||||
t.test("gbnf", test_gbnf_generation);
|
||||
t.test("serialization", test_json_serialization);
|
||||
t.test("python-dict", test_python_dict_parser);
|
||||
|
||||
return t.summary();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user