MaxLSB commited on
Commit
d529c40
·
verified ·
1 Parent(s): d28f0c2

Publish evaluated multilingual code-search checkpoint with protocol and provenance

Browse files
README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: transformers
3
+ pipeline_tag: text-generation
4
+ base_model: openbmb/MiniCPM5-1B
5
+ datasets:
6
+ - lightonai/code-search-Deepseek-Grep-Multi-SFT
7
+ tags:
8
+ - code-search
9
+ - tool-use
10
+ - sft
11
+ ---
12
+ # cs-minicpm5-1b-sft-weighted-grep
13
+
14
+ Full model checkpoint for repository code localization using the **grep** harness.
15
+
16
+ Weighted supervised fine-tuning of `openbmb/MiniCPM5-1B` on 13,550 clean multilingual DeepSeek-V4-Flash-0731 trajectories, using the `weighted` dataset configuration. Injected teacher finish reminders were removed from student training data.
17
+
18
+ Use the system prompt and tool schemas in `student-protocol.json` with the included tokenizer and chat template. The model requires an external harness to execute tool calls; repository snapshots and search indexes are not included. ColGREP models use semantic search; GREP models use lexical search through the terminal.
19
+
20
+ The original training configuration is included for provenance; local paths in that file must be adapted for another environment. `provenance.json` records checkpoint identity and SHA-256 hashes. Model weights are unchanged from the evaluated local checkpoint. Upstream base-model terms apply.
chat_template.jinja ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}{%- if tools %}
2
+ {%- set tool_definitions %}
3
+ {{- "# Tools\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
4
+ {%- for tool in tools %}
5
+ {{- "\n" }}
6
+ {{- tool | tojson(ensure_ascii=False) }}
7
+ {%- endfor %}
8
+ {{- '\n</tools>\n\nTool usage guidelines:\n- You may call zero or more functions. If no function calls are needed, just answer normally and do not include any <function ... </function>.\n- When calling a function, return an XML object within <function ... </function> using:\n<function name="function-name"><param name="param-name">param-value</param></function>\n- param-value may be multi-line. If it contains <, & or newline characters, wrap it in a CDATA block: <param name="param-name"><![CDATA[...multi-line value...]]></param>' }}
9
+ {%- endset %}
10
+
11
+ {{- '<|im_start|>system\n' }}
12
+ {%- if messages[0].role == 'system' %}
13
+ {%- if '<tool_def_sep>' in messages[0].content %}
14
+ {{- messages[0].content.replace('<tool_def_sep>', tool_definitions) }}
15
+ {%- else %}
16
+ {{- messages[0].content + '\n\n' + tool_definitions }}
17
+ {%- endif %}
18
+ {%- else %}
19
+ {{- tool_definitions.lstrip() }}
20
+ {%- endif %}
21
+ {{- '<|im_end|>\n' }}
22
+ {%- else %}
23
+ {%- if messages[0].role == 'system' %}
24
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
25
+ {%- endif %}
26
+ {%- endif %}
27
+ {%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
28
+ {%- for message in messages[::-1] %}
29
+ {%- set index = (messages|length - 1) - loop.index0 %}
30
+ {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
31
+ {%- set ns.multi_step_tool = false %}
32
+ {%- set ns.last_query_index = index %}
33
+ {%- endif %}
34
+ {%- endfor %}
35
+ {%- for message in messages %}
36
+ {%- if message.content is string %}
37
+ {%- set content = message.content %}
38
+ {%- else %}
39
+ {%- set content = '' %}
40
+ {%- endif %}
41
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
42
+ {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
43
+ {%- elif message.role == "assistant" %}
44
+ {%- set reasoning_content = '' %}
45
+ {%- if message.reasoning_content is string %}
46
+ {%- set reasoning_content = message.reasoning_content %}
47
+ {%- else %}
48
+ {%- if '</think>' in content %}
49
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
50
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
51
+ {%- endif %}
52
+ {%- endif %}
53
+
54
+ {%- if message.tool_calls %}
55
+ {%- set content_parts = content.split('<tool_sep>') %}
56
+ {%- set processed_content = content_parts[0] %}
57
+ {%- set tool_calls_count = message.tool_calls|length %}
58
+ {%- set tool_sep_count = content_parts|length - 1 %}
59
+ {%- set min_count = [tool_calls_count, tool_sep_count]|min %}
60
+
61
+ {%- for i in range(1, content_parts|length) %}
62
+ {%- set tool_index = i - 1 %}
63
+ {%- if tool_index < tool_calls_count %}
64
+ {%- set tool_call = message.tool_calls[tool_index] %}
65
+ {%- if tool_call.function %}
66
+ {%- set tool_call = tool_call.function %}
67
+ {%- endif %}
68
+ {%- set single_tool_xml %}
69
+ {{- '<function name="' ~ tool_call.name ~ '">' }}
70
+ {%- if tool_call.arguments %}
71
+ {%- set args_dict = tool_call.arguments %}
72
+ {%- for param_name, param_value in args_dict.items() %}
73
+ {{- '<param name="' ~ param_name ~ '">' }}
74
+ {%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}
75
+ {{- '<![CDATA[' + param_value + ']]>' }}
76
+ {%- else %}
77
+ {{- param_value }}
78
+ {%- endif %}
79
+ {{- '</param>' }}
80
+ {%- endfor %}
81
+ {%- endif %}
82
+ {{- '</function>' }}
83
+ {%- endset %}
84
+ {%- set processed_content = processed_content + single_tool_xml + content_parts[i] %}
85
+ {%- else %}
86
+ {%- set processed_content = processed_content + content_parts[i] %}
87
+ {%- endif %}
88
+ {%- endfor %}
89
+
90
+ {%- if tool_calls_count > tool_sep_count %}
91
+ {%- for remaining_index in range(tool_sep_count, tool_calls_count) %}
92
+ {%- set tool_call = message.tool_calls[remaining_index] %}
93
+ {%- if tool_call.function %}
94
+ {%- set tool_call = tool_call.function %}
95
+ {%- endif %}
96
+ {%- set remaining_tool_xml %}
97
+ {{- '<function name="' ~ tool_call.name ~ '">' }}
98
+ {%- if tool_call.arguments %}
99
+ {%- set args_dict = tool_call.arguments %}
100
+ {%- for param_name, param_value in args_dict.items() %}
101
+ {{- '<param name="' ~ param_name ~ '">' }}
102
+ {%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}
103
+ {{- '<![CDATA[' + param_value + ']]>' }}
104
+ {%- else %}
105
+ {{- param_value }}
106
+ {%- endif %}
107
+ {{- '</param>' }}
108
+ {%- endfor %}
109
+ {%- endif %}
110
+ {{- '</function>' }}
111
+ {%- endset %}
112
+ {%- set processed_content = processed_content + remaining_tool_xml %}
113
+ {%- endfor %}
114
+ {%- endif %}
115
+
116
+ {%- set content = processed_content %}
117
+ {%- endif %}
118
+
119
+ {%- if loop.index0 > ns.last_query_index %}
120
+ {%- if reasoning_content %}
121
+ {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
122
+ {%- else %}
123
+ {{- '<|im_start|>' + message.role + '\n' + content }}
124
+ {%- endif %}
125
+ {%- else %}
126
+ {{- '<|im_start|>' + message.role + '\n' + content }}
127
+ {%- endif %}
128
+
129
+ {%- if message.tool_calls and not has_tool_sep %}
130
+ {%- for tool_call in message.tool_calls %}
131
+ {%- if (loop.first and content) or (not loop.first) %}
132
+ {{- '\n' }}
133
+ {%- endif %}
134
+ {%- if tool_call.function %}
135
+ {%- set tool_call = tool_call.function %}
136
+ {%- endif %}
137
+ {{- '<function name="' ~ tool_call.name ~ '">' }}
138
+ {%- if tool_call.arguments %}
139
+ {%- set args_dict = tool_call.arguments %}
140
+ {%- for param_name, param_value in args_dict.items() %}
141
+ {{- '<param name="' ~ param_name ~ '">' }}
142
+ {%- if param_value is string and ('<' in param_value or '&' in param_value or '\n' in param_value) %}
143
+ {{- '<![CDATA[' + param_value + ']]>' }}
144
+ {%- else %}
145
+ {{- param_value }}
146
+ {%- endif %}
147
+ {{- '</param>' }}
148
+ {%- endfor %}
149
+ {%- endif %}
150
+ {{- '</function>' }}
151
+ {%- endfor %}
152
+ {%- endif %}
153
+ {{- '<|im_end|>\n' }}
154
+ {%- elif message.role == "tool" %}
155
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
156
+ {{- '<|im_start|>user' }}
157
+ {%- endif %}
158
+ {{- '\n<tool_response>\n' }}
159
+ {%- if message.content is string %}
160
+ {{- content }}
161
+ {%- else %}
162
+ {{- message.content | tojson(ensure_ascii=False) }}
163
+ {%- endif %}
164
+ {{- '\n</tool_response>' }}
165
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
166
+ {{- '<|im_end|>\n' }}
167
+ {%- endif %}
168
+ {%- endif %}
169
+ {%- endfor %}
170
+ {%- if add_generation_prompt %}
171
+ {{- '<|im_start|>assistant\n' }}
172
+ {%- if enable_thinking is defined %}
173
+ {%- if enable_thinking is false %}
174
+ {{- '<think>\n\n</think>\n\n' }}
175
+ {%- elif enable_thinking is true %}
176
+ {{- '<think>\n' }}
177
+ {%- endif %}
178
+ {%- endif %}
179
+ {%- endif %}
config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "LlamaForCausalLM"
4
+ ],
5
+ "attention_bias": false,
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 0,
8
+ "dtype": "bfloat16",
9
+ "eos_token_id": 1,
10
+ "head_dim": 128,
11
+ "hidden_act": "silu",
12
+ "hidden_size": 1536,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 4608,
15
+ "max_position_embeddings": 131072,
16
+ "mlp_bias": false,
17
+ "model_type": "llama",
18
+ "num_attention_heads": 16,
19
+ "num_hidden_layers": 24,
20
+ "num_key_value_heads": 2,
21
+ "pad_token_id": 1,
22
+ "pretraining_tp": 1,
23
+ "rms_norm_eps": 1e-06,
24
+ "rope_parameters": {
25
+ "rope_theta": 5000000,
26
+ "rope_type": "default"
27
+ },
28
+ "tie_word_embeddings": false,
29
+ "transformers_version": "5.12.1",
30
+ "use_cache": false,
31
+ "vocab_size": 130560
32
+ }
generation_config.json ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 0,
4
+ "do_sample": true,
5
+ "eos_token_id": [
6
+ 1,
7
+ 1,
8
+ 130073
9
+ ],
10
+ "pad_token_id": 1,
11
+ "temperature": 0.9,
12
+ "top_p": 0.95,
13
+ "transformers_version": "5.12.1"
14
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:24e46b6739a02e6187017a32dd0587953119d32dba201416cfca00257337d729
3
+ size 2161290944
provenance.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "repo_id": "lightonai/cs-minicpm5-1b-sft-weighted-grep",
3
+ "source_checkpoint": "artifacts/models/grep-rebalanced15k-v2-weighted-minicpm5-1b",
4
+ "stage": "sft",
5
+ "family": "minicpm5-1b",
6
+ "harness": "grep",
7
+ "base_model": "openbmb/MiniCPM5-1B",
8
+ "sft_dataset": "lightonai/code-search-Deepseek-Grep-Multi-SFT",
9
+ "sft_configuration": "weighted",
10
+ "sft_rows": 13550,
11
+ "training_job": 4945,
12
+ "sha256": {
13
+ "config.json": "32a9a3eb015d53197ca899efa1f4d60cf1bb449f6b205e9759c9128ddb96077a",
14
+ "generation_config.json": "038b4cfe9ed17a9f30d4a93c8d55d7ae029800d686033624b1ef7eb7dee3c3ed",
15
+ "model.safetensors": "24e46b6739a02e6187017a32dd0587953119d32dba201416cfca00257337d729",
16
+ "tokenizer.json": "3e065a558a034185fe299917b398685c1facd0169a9eea1e629eb30c171fed81",
17
+ "tokenizer_config.json": "3ad32522c384dbe35192bb69de9befbf3f523e99d4bb3f95da757671d4c28281",
18
+ "chat_template.jinja": "7451a05cf1e28a79d97d7c0bc951028c0b1915119bf9046acd06a0e3d931f47c"
19
+ }
20
+ }
student-protocol.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "system_prompt": "You are a code-search agent. Given a software problem statement, find the exact code locations \u2014 files and line ranges \u2014 a developer must read and edit to solve it. This is a RETRIEVAL task: locate the relevant context; do NOT write a fix.\n\nFollow this method:\n\u2022 Start from the task's concrete tokens: grep for identifiers, error strings, function or class names, and other literal text the problem mentions (e.g. `grep -rn \"TokenError\" .`).\n\u2022 Narrow down: refine patterns with path scopes or file filters to isolate the most relevant code.\n\u2022 Read and verify: open promising files and read the exact lines to confirm they are the right locations.\n\u2022 Iterate: if results are too broad or off-target, reformulate your pattern from a different angle and search again.\n\nYou MUST search the repository before answering. Never answer from prior knowledge.\n\n### Search policy\nEvery lookup is a `terminal` call; `grep` is your search tool and there is no semantic search. grep matches TEXT: search for the concrete tokens the code must contain \u2014 identifiers, error strings, option names, function or class names from the problem statement \u2014 never a paraphrase of what the code does.\n- Start repository-wide: terminal {\"command\": \"grep -rn \\\"ask_yesno\\\" .\"}. Use `-i` for case-insensitive, `-w` for whole words, `-l` to list files only, `--include=*.py` to restrict the file type.\n- Search several angles: the symptom, the identifier, its callers, the feature name.\n- Narrow down: once results point at a directory or file, scope the next grep to it, and add `-A`/`-B` context lines to read around a hit.\n- Too many hits? Add a path or `--include`. None? Shorten the pattern to a rarer sub-token, try the wording the code would use, or `find . -name` the file.\n- Read and verify with cat, head, tail or `sed -n START,ENDp` before answering: the locations you submit must be lines you have seen.\n\n### Finishing\nWhen you have identified the relevant locations, you MUST use the `finish` tool to submit your answer. Do NOT simply reply with text \u2014 call the tool.\nThe `finish` tool expects a `locations` list; each entry is a string:\n- `path/to/file`\n- `path/to/file:START-END` (line range)\n- `path/to/file:ClassName.method_name` (symbol)\nCall `finish` once when done. Do not keep searching after you know the answer.\nComplete the task within at most 10 assistant responses. Call finish as soon as you have identified the relevant locations.",
3
+ "tools": [
4
+ {
5
+ "type": "function",
6
+ "function": {
7
+ "name": "terminal",
8
+ "description": "The ONLY tool for exploring the repository: run one read-only shell command. Search with `grep -rn PATTERN [PATH] [--include=GLOB]` (also `grep -rl`, `find`); read with cat, head, tail and `sed -n START,ENDp`; navigate with ls, cd, pwd. There is no semantic search: match the concrete tokens the code contains.",
9
+ "parameters": {
10
+ "type": "object",
11
+ "properties": {
12
+ "command": {
13
+ "type": "string",
14
+ "description": "One shell command, e.g. `grep -rn \"TokenError\" src/ --include=*.py`, `grep -rl \"def validate_expiry\" .`, `sed -n 80,140p src/auth/session.py`."
15
+ }
16
+ },
17
+ "required": [
18
+ "command"
19
+ ]
20
+ }
21
+ }
22
+ },
23
+ {
24
+ "type": "function",
25
+ "function": {
26
+ "name": "finish",
27
+ "description": "Submit your final answer: the code locations a developer must read and edit. Call this once, when you are done searching. Each location is a string: 'path/to/file', 'path/to/file:START-END' (line range), or 'path/to/file:ClassName.method_name' (symbol).",
28
+ "parameters": {
29
+ "type": "object",
30
+ "properties": {
31
+ "locations": {
32
+ "type": "array",
33
+ "items": {
34
+ "type": "string"
35
+ },
36
+ "description": "The relevant locations, most important first."
37
+ }
38
+ },
39
+ "required": [
40
+ "locations"
41
+ ]
42
+ }
43
+ }
44
+ }
45
+ ],
46
+ "max_responses": 10,
47
+ "reminder": false,
48
+ "final_response_tools": [
49
+ "finish"
50
+ ],
51
+ "extra_finish_turns": 0,
52
+ "rescue": "none"
53
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": null,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "is_local": false,
8
+ "legacy": true,
9
+ "local_files_only": false,
10
+ "model_max_length": 1000000000000000019884624838656,
11
+ "pad_token": "</s>",
12
+ "sp_model_kwargs": {},
13
+ "spaces_between_special_tokens": false,
14
+ "tokenizer_class": "TokenizersBackend",
15
+ "unk_token": "<unk>",
16
+ "use_default_system_prompt": false
17
+ }
training-config.yaml ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ name: openbmb/MiniCPM5-1B
3
+ attn_implementation: flash_attention_3
4
+ chat_template: /mnt/vast/maxence/code-search/configuration/training/templates/minicpm5_generation.jinja
5
+ datasets:
6
+ - name: /mnt/vast/maxence/code-search/artifacts/training/rebalanced15k-v2/grep/weighted.jsonl
7
+ split: train
8
+ validation:
9
+ split_ratio: 0.0
10
+ seed: 42
11
+ training:
12
+ output_dir: /mnt/vast/maxence/code-search/artifacts/models/grep-rebalanced15k-v2-weighted-minicpm5-1b
13
+ num_train_epochs: 1
14
+ per_device_train_batch_size: 4
15
+ gradient_accumulation_steps: 8
16
+ learning_rate: 2.0e-05
17
+ lr_scheduler_type: cosine
18
+ warmup_ratio: 0.03
19
+ weight_decay: 0.0
20
+ max_grad_norm: 1.0
21
+ bf16: true
22
+ max_seq_length: 32768
23
+ packing: true
24
+ assistant_only_loss: true
25
+ gradient_checkpointing: true
26
+ use_liger_kernel: true
27
+ optim: adamw_torch
28
+ logging_steps: 10
29
+ save_strategy: steps
30
+ save_steps: 500
31
+ eval_strategy: 'no'
32
+ report_to: none
33
+ dataset_num_proc: 8
34
+ wandb:
35
+ enabled: false
36
+ project: code-search-sft
37
+ run_name: grep-rebalanced15k-v2-weighted-minicpm5-1b