DeepSeek-V4.1-Flash-1017-UD-IQ2XS-GGUF / llama-quant-dither-zeros.patch
cliecy's picture
Add files using upload-large-folder tool
42874c7 verified
Raw History Blame Contribute Delete
2.28 kB
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -1328,6 +1328,33 @@
f32_data = (const float *) f32_conv_buf.data();
}
+ // LLAMA_QUANT_DITHER_ZEROS=1: the IQ2 grids have no zero level, and the IQ2 quantizers give
+ // x == 0 a positive sign. A source with many exact zeros (MXFP4 experts: ~12 %) then gets the
+ // same positive error at every zero, which adds up along each row (measured for IQ2_XS and
+ // IQ2_S). A tiny pseudo-random sign on the zeros turns that into unbiased noise. IQ3_XXS/IQ3_S
+ // also lack a zero level and are included, but were not measured.
+ static const bool dither_zeros = getenv("LLAMA_QUANT_DITHER_ZEROS") && atoi(getenv("LLAMA_QUANT_DITHER_ZEROS"));
+ if (dither_zeros && (
+ new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_S ||
+ new_type == GGML_TYPE_IQ3_XXS || new_type == GGML_TYPE_IQ3_S)) {
+ if (f32_data != (const float *) f32_conv_buf.data()) {
+ if (f32_conv_buf.size() < (size_t) nelements_cur) {
+ f32_conv_buf.resize(nelements_cur);
+ }
+ memcpy(f32_conv_buf.data(), f32_data, nelements_cur*sizeof(float));
+ f32_data = (const float *) f32_conv_buf.data();
+ }
+ float * x = (float *) f32_conv_buf.data();
+ const uint64_t base = (uint64_t) ir * n_per_row;
+ for (int64_t j = 0; j < nelements_cur; ++j) {
+ if (x[j] == 0.0f) {
+ uint64_t h = (base + j + 1) * 0x9E3779B97F4A7C15ULL;
+ h ^= h >> 31;
+ x[j] = (h & 1) ? 1e-20f : -1e-20f;
+ }
+ }
+ }
+
if (work.size() < nrows_cur*row_size_dst) {
work.resize(nrows_cur*row_size_dst);
}