prometheusAIR commited on
Commit
8f39b96
·
verified ·
1 Parent(s): 019971d

Add patches/dsv41s-runtime.patch

Browse files
Files changed (1) hide show
  1. patches/dsv41s-runtime.patch +244 -0
patches/dsv41s-runtime.patch ADDED
@@ -0,0 +1,244 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ DeepSeek-V4.1 runtime patch for the DeepSeek-V4.1-Flash-1017 GGUFs in this repository.
2
+
3
+ Apply on top of smalinin/llama.cpp, branch my_build_deepseek41, commit cfd8adcf6:
4
+
5
+ git clone -b my_build_deepseek41 https://github.com/smalinin/llama.cpp llama.cpp-dsv41
6
+ cd llama.cpp-dsv41 && git checkout cfd8adcf6
7
+ git apply dsv41s-runtime.patch
8
+
9
+ Three changes, squashed into one diff:
10
+
11
+ 1. CUDA: pad MUL_MAT_ID MMQ src1 for the widest selectable tile (fixes an Xid 31 fault in the routed-expert
12
+ matmul). Author: Nick Mermiges. The ids path sized the src1 tail padding from ne11 (1 for broadcast
13
+ gate/up, n_expert_used for down), so no padding was added, and the last tile of the last expert read up to
14
+ J-1 columns past the buffer. When that buffer ended at a fresh VMM pool high-water mark, the read hit unmapped
15
+ memory.
16
+ 2. tests: a test-backend-ops reproducer for (1). Author: Nick Mermiges.
17
+ 3. dsv41s: (a) also recognise V4.1 chat templates that build the tool-call opener as "<' + dsml_token + ' calls>"
18
+ (pfeifferj's, midagedev's and ours), not only smalinin's own marker string; (b) load GGUFs that use the older
19
+ pfeifferj engram metadata layout. Neither is needed for the files in this repo, whose embedded template
20
+ carries smalinin's marker and whose engram metadata is the current layout. They are kept so this is exactly
21
+ the tree the files were validated on. Author: prometheusAIR.
22
+
23
+ diff --git a/common/chat.cpp b/common/chat.cpp
24
+ index 254440a89..926e03d2f 100644
25
+ --- a/common/chat.cpp
26
+ +++ b/common/chat.cpp
27
+ @@ -2171,7 +2171,11 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha
28
+ // and " parameter". Both V4 variants render tool results in tool call order and use
29
+ // a bare </think> for the non-thinking generation prompt.
30
+ const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
31
+ - const bool is_v4_1 = tmpl.source().find("tool_calls_block_name: V4.1") != std::string::npos;
32
+ + // smalinin's template carries a marker; other V4.1 templates (pfeifferj's, midagedev's) build the
33
+ + // opener as "<' + dsml_token + ' calls>", so that literal identifies them too.
34
+ + const bool is_v4_1 = tmpl.source().find("tool_calls_block_name: V4.1") != std::string::npos ||
35
+ + (is_v4 && (tmpl.source().find("' calls>") != std::string::npos ||
36
+ + tmpl.source().find("\" calls>") != std::string::npos));
37
+
38
+ std::optional<json> adjusted_messages;
39
+ if (is_v4) {
40
+ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
41
+ index 7fb440148..9cff3bcb3 100644
42
+ --- a/ggml/src/ggml-cuda/mmq.cu
43
+ +++ b/ggml/src/ggml-cuda/mmq.cu
44
+ @@ -202,8 +202,11 @@ void ggml_cuda_mul_mat_q(
45
+ CUDA_CHECK(cudaGetLastError());
46
+ }
47
+
48
+ + // MMQ loads whole J-column src1 tiles and J is picked against ncols_opt (tokens), not ne11 (1 or n_expert_used),
49
+ + // so the last tile of the last expert reads up to J-1 columns past the compact src1. Pad for the widest tile:
50
+ + // sizing this from ne11 gave zero padding and the read ran off the end of the VMM pool (Xid 31).
51
+ const size_t nbytes_src1_q8_1 = ne12*n_expert_used*ne10_padded * y_block_size/y_values_per_block +
52
+ - ggml_cuda_mmq_get_J_max(src0->type, fallback, cc, ne11) * sizeof(block_q8_1_mmq);
53
+ + ggml_cuda_mmq_get_J_max(src0->type, fallback, cc, MMQ_J_SELECT_MAX) * sizeof(block_q8_1_mmq);
54
+ ggml_cuda_pool_alloc<char> src1_q8_1(ctx.pool(), nbytes_src1_q8_1);
55
+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool());
56
+ if (src0->type == GGML_TYPE_NVFP4 && use_native_fp4) {
57
+ diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
58
+ index b4a747720..be205a4d6 100644
59
+ --- a/ggml/src/ggml-cuda/mmq.cuh
60
+ +++ b/ggml/src/ggml-cuda/mmq.cuh
61
+ @@ -363,6 +363,9 @@ static constexpr __device__ int ggml_cuda_mmq_get_sram_stride(ggml_type type, in
62
+ return ggml_cuda_mmq_get_sram_stride(ggml_cuda_mmq_get_sram_layout(type, J, fallback));
63
+ }
64
+
65
+ +// Widest tile in the src1/dst column direction that mul_mat_q_switch_J may select.
66
+ +static constexpr int MMQ_J_SELECT_MAX = 128;
67
+ +
68
+ static __host__ int ggml_cuda_mmq_get_J_max(const ggml_type type, const bool fallback, const int cc, const int64_t ne11) {
69
+ int ret = std::min(ne11, int64_t(512));
70
+ ret -= ret % 8;
71
+ @@ -1476,7 +1479,7 @@ void mul_mat_q_switch_J(ggml_backend_cuda_context & ctx, const mmq_args & args,
72
+ int J_best = 0;
73
+ int ntiles_J_best = INT_MAX;
74
+
75
+ - for (int J = 8; J <= 128 && ntiles_J_best > 1; J += 8) {
76
+ + for (int J = 8; J <= MMQ_J_SELECT_MAX && ntiles_J_best > 1; J += 8) {
77
+ const ggml_cuda_mmq_config config = ggml_cuda_mmq_get_config(type, J, fallback, cc);
78
+ if (config.type == GGML_TYPE_COUNT) {
79
+ continue;
80
+ diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
81
+ index 78c696d4c..23ec8731f 100644
82
+ --- a/src/llama-model-loader.cpp
83
+ +++ b/src/llama-model-loader.cpp
84
+ @@ -408,6 +408,8 @@ namespace GGUFMeta {
85
+ return get_arr(llm_kv(kid), result, required);
86
+ }
87
+
88
+ + template bool llama_model_loader::get_arr<uint32_t, LLAMA_MAX_LAYERS>(const std::string & key, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);
89
+ +
90
+ template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);
91
+ template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);
92
+ template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);
93
+ diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
94
+ index 8ea370c92..53811a21b 100644
95
+ --- a/src/models/deepseek41.cpp
96
+ +++ b/src/models/deepseek41.cpp
97
+ @@ -144,16 +144,21 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) {
98
+ hparams.dsv41_candidate_top_k_blocks);
99
+ }
100
+
101
+ - ml.get_arr_n(LLM_KV_ENGRAM_LAYER_IDS, engram_n_layer);
102
+ + // Legacy engram metadata (pfeifferj's GSQ-RCO GGUF, ported from his runtime patch): old key
103
+ + // names, and the token map stored as an I32 tensor rather than a KV array.
104
+ + const bool legacy = ml.get_weight("engram_token_map.weight") != nullptr;
105
+ + const std::string prefix = "deepseek41.engram.";
106
+ + const auto key = [&](const char * modern, const char * old) { return prefix + (legacy ? old : modern); };
107
+ + ml.get_arr_n(key("layer_ids", "layers"), engram_n_layer);
108
+ if (engram_n_layer == 0 || engram_n_layer > LLAMA_MAX_LAYERS) {
109
+ throw std::runtime_error(format("DeepSeek-V4.1 engram layer count %u is out of range", engram_n_layer));
110
+ }
111
+ - ml.get_arr(LLM_KV_ENGRAM_LAYER_IDS, hparams.engram_layer_ids);
112
+ + ml.get_arr(key("layer_ids", "layers"), hparams.engram_layer_ids);
113
+
114
+ ml.get_key(LLM_KV_ENGRAM_HEAD_COUNT, hparams.engram_n_head);
115
+ - ml.get_key(LLM_KV_ENGRAM_KEY_LENGTH, hparams.engram_key_length);
116
+ - ml.get_key(LLM_KV_ENGRAM_MAX_NGRAM_SIZE, hparams.engram_max_ngram_size);
117
+ - ml.get_key(LLM_KV_ENGRAM_PAD_ID, engram_pad_id);
118
+ + ml.get_key(key("key_length", "head_dim"), hparams.engram_key_length);
119
+ + ml.get_key(key("max_ngram_size", "ngram_size"), hparams.engram_max_ngram_size);
120
+ + ml.get_key(key("pad_id", "pad_token_id"), engram_pad_id);
121
+
122
+ if (hparams.engram_n_head == 0 || hparams.engram_max_ngram_size < 2) {
123
+ throw std::runtime_error("DeepSeek-V4.1 engram needs at least one head and a 2-gram");
124
+ @@ -168,7 +173,30 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) {
125
+ ml.get_arr(LLM_KV_ENGRAM_MULTIPLIERS, engram_multipliers);
126
+ ml.get_arr(LLM_KV_ENGRAM_PRIMES, engram_primes);
127
+ ml.get_arr(LLM_KV_ENGRAM_OFFSETS, engram_offsets);
128
+ - ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map);
129
+ + if (legacy) {
130
+ + const auto * w = ml.get_weight("engram_token_map.weight");
131
+ + if (w->tensor->type != GGML_TYPE_I32 || w->tensor->ne[1] != 1 || w->tensor->ne[2] != 1 || w->tensor->ne[3] != 1) {
132
+ + throw std::runtime_error("engram_token_map.weight must be a one-dimensional I32 tensor");
133
+ + }
134
+ + std::vector<int32_t> token_map(w->tensor->ne[0]);
135
+ + ml.files.at(w->idx)->seek(w->offs, SEEK_SET);
136
+ + ml.files.at(w->idx)->read_raw(token_map.data(), token_map.size()*sizeof(int32_t));
137
+ + uint32_t compressed_vocab = 0;
138
+ + ml.get_key(prefix + "compressed_vocab_size", compressed_vocab);
139
+ + for (int32_t id : token_map) {
140
+ + if (id < 0 || (uint32_t) id >= compressed_vocab) {
141
+ + throw std::runtime_error("Engram token map index is outside the compressed vocabulary");
142
+ + }
143
+ + }
144
+ + engram_token_map.assign(token_map.begin(), token_map.end());
145
+ + if (engram_pad_id >= engram_token_map.size()) {
146
+ + throw std::runtime_error("Engram padding token is outside the token map");
147
+ + }
148
+ + // the legacy file stores the raw pad token; the modern key stores it already mapped
149
+ + engram_pad_id = engram_token_map[engram_pad_id];
150
+ + } else {
151
+ + ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map);
152
+ + }
153
+
154
+ // the hash indexes straight into these, so a short array would read past the end
155
+ const size_t n_bucket = (size_t) (hparams.engram_max_ngram_size - 1) * hparams.engram_n_head;
156
+ @@ -189,6 +217,10 @@ void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) {
157
+
158
+ void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) {
159
+ LLAMA_LOAD_LOCALS;
160
+ + if (ml.get_weight("engram_token_map.weight")) {
161
+ + // the legacy token map was read and validated in load_arch_hparams
162
+ + ml.n_created++;
163
+ + }
164
+
165
+ const int64_t q_lora_rank = hparams.n_lora_q;
166
+ const int64_t n_ff_exp = hparams.n_ff_exp();
167
+ diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
168
+ index 0499d5923..3e38f410c 100644
169
+ --- a/tests/test-backend-ops.cpp
170
+ +++ b/tests/test-backend-ops.cpp
171
+ @@ -5026,6 +5026,61 @@ struct test_mul_mat_id : public test_case {
172
+ }
173
+ };
174
+
175
+ +// GGML_OP_MUL_MAT_ID, MMQ tail tile of the last expert.
176
+ +// MMQ loads whole J-column src1 tiles, so the last tile of the last expert reads past the compact quantized src1.
177
+ +// Fixed routing (token t uses experts t*n_used .. t*n_used + 3, mod 1000) leaves 3 tokens on expert 999, and the
178
+ +// sizes make the pool allocations 14336 + 14336 + 4096 + 2064384 bytes = exactly one 2 MiB VMM granule, so a src1
179
+ +// without tail padding ends on unmapped memory and the 128-column tile faults (Xid 31) instead of reading stale
180
+ +// pool bytes. Needs a fresh pool: run it alone, e.g. test-backend-ops -o MUL_MAT_ID -p y_tail.
181
+ +struct test_mul_mat_id_y_tail : public test_case {
182
+ + static constexpr int n_mats = 1000;
183
+ + static constexpr int n_used = 4;
184
+ + static constexpr int64_t m = 128;
185
+ + static constexpr int64_t n = 896;
186
+ + static constexpr int64_t k = 512;
187
+ + const bool b; // broadcast b matrix
188
+ +
189
+ + std::string vars() override {
190
+ + return "y_tail," + VARS_TO_STR1(b);
191
+ + }
192
+ +
193
+ + double max_nmse_err() override {
194
+ + return 5e-4;
195
+ + }
196
+ +
197
+ + test_mul_mat_id_y_tail(bool b) : b(b) {}
198
+ +
199
+ + ggml_tensor * build_graph(ggml_context * ctx) override {
200
+ + ggml_tensor * as = ggml_new_tensor_3d(ctx, GGML_TYPE_MXFP4, k, m, n_mats);
201
+ + ggml_set_name(as, "as");
202
+ +
203
+ + ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_used, n);
204
+ + ggml_set_name(ids, "ids");
205
+ +
206
+ + ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, k, this->b ? 1 : n_used, n);
207
+ + ggml_set_name(b, "b");
208
+ +
209
+ + ggml_tensor * out = ggml_mul_mat_id(ctx, as, b, ids);
210
+ + ggml_set_name(out, "out");
211
+ +
212
+ + return out;
213
+ + }
214
+ +
215
+ + void initialize_tensors(ggml_context * ctx) override {
216
+ + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
217
+ + if (t->type != GGML_TYPE_I32) {
218
+ + init_tensor_uniform(t);
219
+ + continue;
220
+ + }
221
+ + std::vector<int32_t> data(n_used*n);
222
+ + for (size_t i = 0; i < data.size(); i++) {
223
+ + data[i] = i % n_mats;
224
+ + }
225
+ + ggml_backend_tensor_set(t, data.data(), 0, ggml_nbytes(t));
226
+ + }
227
+ + }
228
+ +};
229
+ +
230
+ // GGML_OP_MUL_MAT_ID + GGML_OP_ADD or GGML_OP_MUL
231
+ struct test_mul_mat_id_fusion : public test_case {
232
+ const ggml_type type_a;
233
+ @@ -10368,6 +10423,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
234
+ }
235
+ test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3));
236
+
237
+ + // DeepSeek V4.1 Xid 31: MMQ tail tile past the compact src1 of MUL_MAT_ID
238
+ + for (bool b : {true, false}) {
239
+ + test_cases.emplace_back(new test_mul_mat_id_y_tail(b));
240
+ + }
241
+ +
242
+ // gpt-oss issue with Vulkan mmq_id
243
+ test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_MXFP4, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880));
244
+ test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_0, GGML_TYPE_F32, 32, 2, false, 2880, 32, 2880));