mikhail/llama.cpp-mtp-turboquant
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

jinja : implement mixed type object keys (#18955)

Browse Source 
* implement mixed type object keys

* add tests

* refactor

* minor fixes

* massive refactor

* add more tests

* forgotten tuples

* fix array/object is_hashable

* correct (albeit broken) jinja responses

verified with transformers

* improved hashing and equality

* refactor hash function

* more exhausive test case

* clean up

* cont

* cont (2)

* missing cstring

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
Sigbjørn Skjæret
2026-01-27 19:50:42 +01:00
committed by GitHub
parent 68ac3acb43
commit 2b4cbd2834
9 changed files with 802 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-jinja.cpp
+217
View File
@@ -9,6 +9,7 @@
#include "jinja/runtime.h"
#include "jinja/parser.h"
#include "jinja/lexer.h"
#include "jinja/utils.h"
#include "testing.h"
@@ -30,6 +31,7 @@ static void test_tests(testing & t);
static void test_string_methods(testing & t);
static void test_array_methods(testing & t);
static void test_object_methods(testing & t);
static void test_hasher(testing & t);
static void test_fuzzing(testing & t);
static bool g_python_mode = false;
@@ -67,6 +69,7 @@ int main(int argc, char *argv[]) {
t.test("array methods", test_array_methods);
t.test("object methods", test_object_methods);
if (!g_python_mode) {
t.test("hasher", test_hasher);
t.test("fuzzing", test_fuzzing);
}
@@ -156,6 +159,18 @@ static void test_conditionals(testing & t) {
"big"
);
test_template(t, "object comparison",
"{% if {0: 1, none: 2, 1.0: 3, '0': 4, true: 5} == {false: 1, none: 2, 1: 5, '0': 4} %}equal{% endif %}",
json::object(),
"equal"
);
test_template(t, "array comparison",
"{% if [0, 1.0, false] == [false, 1, 0.0] %}equal{% endif %}",
json::object(),
"equal"
);
test_template(t, "logical and",
"{% if a and b %}both{% endif %}",
{{"a", true}, {"b", true}},
@@ -358,6 +373,30 @@ static void test_expressions(testing & t) {
"b"
);
test_template(t, "array negative access",
"{{ items[-1] }}",
{{"items", json::array({"a", "b", "c"})}},
"c"
);
test_template(t, "array slice",
"{{ items[1:-1]|string }}",
{{"items", json::array({"a", "b", "c"})}},
"['b']"
);
test_template(t, "array slice step",
"{{ items[::2]|string }}",
{{"items", json::array({"a", "b", "c"})}},
"['a', 'c']"
);
test_template(t, "tuple slice",
"{{ ('a', 'b', 'c')[::-1]|string }}",
json::object(),
"('c', 'b', 'a')"
);
test_template(t, "arithmetic",
"{{ (a + b) * c }}",
{{"a", 2}, {"b", 3}, {"c", 4}},
@@ -401,6 +440,36 @@ static void test_set_statement(testing & t) {
json::object(),
"1"
);
test_template(t, "set dict with mixed type keys",
"{% set d = {0: 1, none: 2, 1.0: 3, '0': 4, (0, 0): 5, false: 6, 1: 7} %}{{ d[(0, 0)] + d[0] + d[none] + d['0'] + d[false] + d[1.0] + d[1] }}",
json::object(),
"37"
);
test_template(t, "print dict with mixed type keys",
"{% set d = {0: 1, none: 2, 1.0: 3, '0': 4, (0, 0): 5, true: 6} %}{{ d|string }}",
json::object(),
"{0: 1, None: 2, 1.0: 6, '0': 4, (0, 0): 5}"
);
test_template(t, "print array with mixed types",
"{% set d = [0, none, 1.0, '0', true, (0, 0)] %}{{ d|string }}",
json::object(),
"[0, None, 1.0, '0', True, (0, 0)]"
);
test_template(t, "object member assignment with mixed key types",
"{% set d = namespace() %}{% set d.a = 123 %}{{ d['a'] == 123 }}",
json::object(),
"True"
);
test_template(t, "tuple unpacking",
"{% set t = (1, 2, 3) %}{% set a, b, c = t %}{{ a + b + c }}",
json::object(),
"6"
);
}
static void test_filters(testing & t) {
@@ -1312,6 +1381,154 @@ static void test_object_methods(testing & t) {
{{"obj", {{"a", "b"}}}},
"True True"
);
test_template(t, "expression as object key",
"{% set d = {'ab': 123} %}{{ d['a' + 'b'] == 123 }}",
json::object(),
"True"
);
test_template(t, "numeric as object key (template: Seed-OSS)",
"{% set d = {1: 'a', 2: 'b'} %}{{ d[1] == 'a' and d[2] == 'b' }}",
json::object(),
"True"
);
}
static void test_hasher(testing & t) {
static const std::vector<std::pair<size_t, size_t>> chunk_sizes = {
{1, 2},
{1, 16},
{8, 1},
{1, 1024},
{5, 512},
{16, 256},
{45, 122},
{70, 634},
};
static auto random_bytes = [](size_t length) -> std::string {
std::string data;
data.resize(length);
for (size_t i = 0; i < length; ++i) {
data[i] = static_cast<char>(rand() % 256);
}
return data;
};
t.test("state unchanged with empty input", [](testing & t) {
jinja::hasher hasher;
hasher.update("some data");
size_t initial_state = hasher.digest();
hasher.update("", 0);
size_t final_state = hasher.digest();
t.assert_true("Hasher state should remain unchanged", initial_state == final_state);
});
t.test("different inputs produce different hashes", [](testing & t) {
jinja::hasher hasher1;
hasher1.update("data one");
size_t hash1 = hasher1.digest();
jinja::hasher hasher2;
hasher2.update("data two");
size_t hash2 = hasher2.digest();
t.assert_true("Different inputs should produce different hashes", hash1 != hash2);
});
t.test("same inputs produce same hashes", [](testing & t) {
jinja::hasher hasher1;
hasher1.update("consistent data");
size_t hash1 = hasher1.digest();
jinja::hasher hasher2;
hasher2.update("consistent data");
size_t hash2 = hasher2.digest();
t.assert_true("Same inputs should produce same hashes", hash1 == hash2);
});
t.test("property: update(a ~ b) == update(a).update(b)", [](testing & t) {
for (const auto & [size1, size2] : chunk_sizes) {
std::string data1 = random_bytes(size1);
std::string data2 = random_bytes(size2);
jinja::hasher hasher1;
hasher1.update(data1);
hasher1.update(data2);
size_t hash1 = hasher1.digest();
jinja::hasher hasher2;
hasher2.update(data1 + data2);
size_t hash2 = hasher2.digest();
t.assert_true(
"Hashing in multiple updates should match single update (" + std::to_string(size1) + ", " + std::to_string(size2) + ")",
hash1 == hash2);
}
});
t.test("property: update(a ~ b) == update(a).update(b) with more update passes", [](testing & t) {
static const std::vector<size_t> sizes = {3, 732, 131, 13, 17, 256, 436, 99, 4};
jinja::hasher hasher1;
jinja::hasher hasher2;
std::string combined_data;
for (size_t size : sizes) {
std::string data = random_bytes(size);
hasher1.update(data);
combined_data += data;
}
hasher2.update(combined_data);
size_t hash1 = hasher1.digest();
size_t hash2 = hasher2.digest();
t.assert_true(
"Hashing in multiple updates should match single update with many chunks",
hash1 == hash2);
});
t.test("property: non associativity of update", [](testing & t) {
for (const auto & [size1, size2] : chunk_sizes) {
std::string data1 = random_bytes(size1);
std::string data2 = random_bytes(size2);
jinja::hasher hasher1;
hasher1.update(data1);
hasher1.update(data2);
size_t hash1 = hasher1.digest();
jinja::hasher hasher2;
hasher2.update(data2);
hasher2.update(data1);
size_t hash2 = hasher2.digest();
t.assert_true(
"Hashing order should matter (" + std::to_string(size1) + ", " + std::to_string(size2) + ")",
hash1 != hash2);
}
});
t.test("property: different lengths produce different hashes (padding block size)", [](testing & t) {
std::string random_data = random_bytes(64);
jinja::hasher hasher1;
hasher1.update(random_data);
size_t hash1 = hasher1.digest();
for (int i = 0; i < 16; ++i) {
random_data.push_back('A'); // change length
jinja::hasher hasher2;
hasher2.update(random_data);
size_t hash2 = hasher2.digest();
t.assert_true("Different lengths should produce different hashes (length " + std::to_string(random_data.size()) + ")", hash1 != hash2);
hash1 = hash2;
}
});
}
static void test_template_cpp(testing & t, const std::string & name, const std::string & tmpl, const json & vars, const std::string & expect) {