--- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -1328,6 +1328,33 @@ f32_data = (const float *) f32_conv_buf.data(); } + // LLAMA_QUANT_DITHER_ZEROS=1: the IQ2 grids have no zero level, and the IQ2 quantizers give + // x == 0 a positive sign. A source with many exact zeros (MXFP4 experts: ~12 %) then gets the + // same positive error at every zero, which adds up along each row (measured for IQ2_XS and + // IQ2_S). A tiny pseudo-random sign on the zeros turns that into unbiased noise. IQ3_XXS/IQ3_S + // also lack a zero level and are included, but were not measured. + static const bool dither_zeros = getenv("LLAMA_QUANT_DITHER_ZEROS") && atoi(getenv("LLAMA_QUANT_DITHER_ZEROS")); + if (dither_zeros && ( + new_type == GGML_TYPE_IQ2_XXS || new_type == GGML_TYPE_IQ2_XS || new_type == GGML_TYPE_IQ2_S || + new_type == GGML_TYPE_IQ3_XXS || new_type == GGML_TYPE_IQ3_S)) { + if (f32_data != (const float *) f32_conv_buf.data()) { + if (f32_conv_buf.size() < (size_t) nelements_cur) { + f32_conv_buf.resize(nelements_cur); + } + memcpy(f32_conv_buf.data(), f32_data, nelements_cur*sizeof(float)); + f32_data = (const float *) f32_conv_buf.data(); + } + float * x = (float *) f32_conv_buf.data(); + const uint64_t base = (uint64_t) ir * n_per_row; + for (int64_t j = 0; j < nelements_cur; ++j) { + if (x[j] == 0.0f) { + uint64_t h = (base + j + 1) * 0x9E3779B97F4A7C15ULL; + h ^= h >> 31; + x[j] = (h & 1) ? 1e-20f : -1e-20f; + } + } + } + if (work.size() < nrows_cur*row_size_dst) { work.resize(nrows_cur*row_size_dst); }