KV head count

pr_number	parameters
30	{"heads":12,"kv_heads":6}
34	{"num_heads":8,"num_kv_heads":4}
39	{"kv_heads":4}
41	{"num_heads":8,"num_kv_heads":4}
46	{"layers":9,"dim":432,"heads":8,"kv_heads":2,"mlp_mult":2}
48	{"num_heads":8,"num_kv_heads":4,"layers":9,"model_dim":512}
49	{"num_heads":8,"num_kv_heads":4}
50	{"num_heads":8,"num_kv_heads":4}
53	{"num_heads":8,"num_kv_heads":4}
56	{"layers":14,"model_dim":416,"num_heads":8,"num_kv_heads":2,"mlp_mult":2}
65	{"heads":8,"kv_heads":4}
69	{"heads":8,"kv_heads":4}
70	{"num_heads":8,"num_kv_heads":4}
71	{"num_heads":8,"num_kv_heads":4}
74	{"num_heads":8,"num_kv_heads":4}
75	{"num_heads":8,"num_kv_heads":4}
76	{"num_heads":8,"num_kv_heads":4}
79	{"heads":24,"kv_heads":12}
81	{"heads":8,"kv_heads":4}
86	{"attention_heads":8,"kv_heads":4}
94	{"layers":9,"model_dim":512,"num_heads":8,"num_kv_heads":4,"mlp_mult":2}
111	{"layers":9,"model_dim":512,"num_heads":8,"num_kv_heads":4,"mlp_mult":2,"vocab_size":1024}
114	{"layers":9,"dim":512,"heads":8,"kv_heads":4}
125	{"num_heads":8,"num_kv_heads":4}
126	{"heads":8,"kv_heads":4}
136	{"heads":8,"kv_heads":4}
139	{"heads":12,"kv_heads":6}
143	{"num_heads":8,"num_kv_heads":4}
144	{"heads":8,"kv_heads":4}
146	{"num_heads":8,"num_kv_heads":4}
147	{"num_heads":8,"num_kv_heads":4}
148	{"heads":8,"kv_heads":4}
150	{"heads":8,"kv_heads":4}
156	{"layers":9,"model_dim":512,"attention_heads":8,"kv_heads":4}
157	{"layers":9,"model_dim":512,"num_heads":8,"num_kv_heads":4}
159	{"layers":9,"model_dim":512,"heads":8,"kv_heads":4}
160	{"num_heads":8,"num_kv_heads":4}
161	{"layers":9,"dimensions":512,"heads":8,"kv_heads":4}
163	{"num_heads":8,"num_kv_heads":4}
168	{"layers":7,"dim":384,"heads":6,"kv_heads":3}
169	{"heads":8,"kv_heads":4}
173	{"heads":8,"kv_heads":4,"layers":9,"dim":512}
180	{"heads":8,"kv_heads":4}
185	{"heads":12,"kv_heads":4}
190	{"num_heads":8,"num_kv_heads":4}
191	{"num_heads":8,"num_kv_heads":4}
192	{"heads":8,"kv_heads":4}
193	{"kv_heads":4}
194	{"heads":8,"kv_heads":4}
196	{"num_heads":12,"num_kv_heads":4}
204	{"layers":10,"model_dim":512,"num_heads":8,"num_kv_heads":4,"mlp_hidden":1088}
206	{"heads":8,"kv_heads":4}
211	{"layers":12,"heads":8,"kv_heads":4,"dim":512}
212	{"num_heads":8,"num_kv_heads":4}
215	{"kv_heads":4}
217	{"heads":8,"kv_heads":4}
219	{"heads":8,"kv_heads":4}
223	{"heads":8,"kv_heads":4}
231	{"num_heads":8,"num_kv_heads":4}
236	{"heads":8,"kv_heads":4}
237	{"num_heads":8,"num_kv_heads":4}
238	{"num_heads":8,"num_kv_heads":4}
240	{"num_heads":8,"num_kv_heads":2}
247	{"layers":8,"model_dim":512,"num_heads":8,"num_kv_heads":4}
248	{"num_heads":8,"num_kv_heads":4}
249	—
251	{"heads":8,"kv_heads":4}
256	{"num_heads":8,"num_kv_heads":4}
258	{"layers":7,"model_dim":512,"num_heads":8,"num_kv_heads":4}
264	{"heads":8,"kv_heads":4}
267	{"kv_heads":4,"heads":8}
271	{"num_kv_heads":2}
273	{"heads":8,"kv_heads":4}
276	{"num_heads":8,"num_kv_heads":4}
278	{"heads":8,"kv_heads":4}
281	{"heads":8,"kv_heads":4}
294	{"heads":12,"kv_heads":6}
295	{"heads":8,"kv_heads":4}
296	{"kv_heads":4,"attention_heads":8}
304	{"layers":10,"dim":512,"heads":8,"kv_heads":4}
305	{"heads":8,"kv_heads":4}
306	{"heads":8,"kv_heads":4}
309	{"heads":8,"kv_heads":4}
312	{"num_heads":8,"num_kv_heads":4}
317	{"heads":8,"kv_heads":4}
319	{"heads":8,"kv_heads":4}
325	{"num_heads":10,"num_kv_heads":5}
327	{"heads":8,"kv_heads":4}
331	{"heads":8,"kv_heads":4}
334	{"heads":8,"kv_heads":4}
339	{"heads":8,"kv_heads":4}
343	{"heads":4,"kv_heads":2}
349	{"heads":8,"kv_heads":4}
351	{"attention_heads":8,"kv_heads":4}
352	{"attention_heads":8,"kv_heads":4}
355	{"num_heads":8,"num_kv_heads":4}
362	{"num_heads":8,"num_kv_heads":4}
366	{"heads":8,"kv_heads":4}
369	{"heads":8,"kv_heads":4}
372	{"num_kv_heads":4}
373	{"layers":10,"dim":512,"heads":8,"kv_heads":4}
374	{"heads":8,"kv_heads":4}
376	{"heads":8,"kv_heads":4}
379	{"heads":8,"kv_heads":4}
383	{"heads":8,"kv_heads":4}
385	{"heads":8,"kv_heads":4}
386	{"num_heads":12,"num_kv_heads":4,"model_dim":768}
390	{"heads":8,"kv_heads":4}
391	{"kv_heads":8}
393	{"heads":8,"kv_heads":4}
395	{"heads":8,"kv_heads":8}
398	{"heads":8,"kv_heads":4}
400	{"heads":8,"kv_heads":4}
401	{"heads":8,"kv_heads":4}
406	{"heads":8,"kv_heads":4}
410	{"heads":8,"kv_heads":4}
415	{"heads":8,"kv_heads":4}
417	{"heads":8,"kv_heads":4}
418	{"heads":8,"kv_heads":4}
422	{"kv_heads":4}
433	{"q_heads":16,"kv_heads":4}
434	{"heads":8,"kv_heads":4}
436	{"heads":8,"kv_heads":4}
442	{"heads":8,"kv_heads":4}
443	{"heads":8,"kv_heads":4}
444	{"num_heads":8,"num_kv_heads":4}
446	{"attention_heads":8,"kv_heads":4}
447	{"heads":8,"kv_heads":4}
448	{"num_heads":8,"num_kv_heads":4}
450	{"heads":8,"kv_heads":4}
451	{"heads":8,"kv_heads":4}
455	{"heads":8,"kv_heads":4}
456	{"heads":8,"kv_heads":4}
462	{"heads":8,"kv_heads":8}
465	{"attention_heads":8,"kv_heads":4}
466	{"heads":8,"kv_heads":4}
467	{"heads":8,"kv_heads":4}
469	{"heads":9,"kv_heads":3}
470	{"num_heads":16,"num_kv_heads":8}
474	{"heads":8,"kv_heads":4}
477	{"heads":8,"kv_heads":4}
483	{"heads":8,"kv_heads":4}
485	{"heads":8,"kv_heads":4}
493	{"heads":8,"kv_heads":4}
503	{"heads":8,"kv_heads":4}
508	{"kv_heads":4,"heads":8}
512	{"heads":8,"kv_heads":4}
525	{"heads":8,"kv_heads":4}
528	{"heads":8,"kv_heads":4}
529	{"heads":8,"kv_heads":4}
532	{"heads":8,"kv_heads":4}
545	{"heads":8,"kv_heads":8}
548	{"heads":8,"kv_heads":4}
549	{"heads":8,"kv_heads":4}
563	{"kv_heads":4,"attention_heads":8,"model_dim":512,"layers":10}
568	{"heads":8,"kv_heads":4}
580	{"NUM_KV_HEADS":4,"NUM_HEADS":8}
585	{"heads":8,"kv_heads":8}
588	{"NUM_KV_HEADS":2}
595	{"heads":8,"kv_heads":4}
596	{"attention_heads":8,"kv_heads":4}
622	{"heads":8,"kv_heads":4}
634	{"heads":8,"kv_heads":4}
649	{"attention_heads":8,"kv_heads":4}
661	{"kv_heads":8,"heads":8,"layers":11,"dim":512}
664	{"attention_heads":8,"kv_heads":4}
665	{"heads":8,"kv_heads":4}
666	{"layers":12,"dimensions":768,"heads":12,"kv_heads":6}
678	{"num_heads":8,"num_kv_heads":4}
684	{"heads":8,"kv_heads":4}
691	{"heads":8,"kv_heads":4}
694	{"layers":10,"heads":8,"kv_heads":4,"d_model":512}
700	{"layers":11,"hidden_size":512,"heads":8,"kv_heads":8}
709	{"heads":8,"kv_heads":4,"layers":10,"dim":512}
710	{"heads":8,"kv_heads":4}
716	{"heads":8,"kv_heads":4}
728	{"gqa_heads":8,"kv_heads":4}
734	{"heads":8,"kv_heads":4}
738	{"heads":8,"kv_heads":4}
741	{"kv_heads":4}
746	{"heads":8,"kv_heads":4}
749	{"num_heads":8,"num_kv_heads":4}
755	{"heads":6,"kv_heads":2}
757	{"heads":8,"kv_heads":4}
760	{"heads":8,"kv_heads":4}
773	{"num_heads":10,"num_kv_heads":5}
777	{"heads":8,"kv_heads":4}
793	{"attention_heads":8,"kv_heads":4}
796	{"kv_heads":4,"attention_heads":8}
799	{"num_heads":8,"num_kv_heads":4}
802	{"heads":8,"kv_heads":4}
807	{"heads":8,"kv_heads":4}
809	{"query_heads":8,"kv_heads":4}
811	{"heads":8,"kv_heads":4}
820	{"num_heads":8,"num_kv_heads":4}
835	{"heads":8,"kv_heads":4}
838	{"heads":8,"kv_heads":4}
841	{"heads":8,"kv_heads":4}
854	{"heads":8,"kv_heads":4}
856	{"heads":8,"kv_heads":4}
858	{"heads":8,"kv_heads":4}
920	{"num_kv_heads":4}
941	{"heads":8,"kv_heads":4}
945	{"heads":8,"kv_heads":8}
953	{"heads":8,"kv_heads":8}
967	{"kv_heads":8}
995	{"heads":8,"kv_heads":8}
1025	{"heads":8,"kv_heads":8}
1036	{"heads":8,"kv_heads":8}
1056	{"kv_heads":8}
1059	{"kv_heads":4}
1104	{"byte260":4,"sp1024":2}
1127	{"heads":4}
1139	{"heads":5,"kv_heads":5}
1140	{"heads":8,"kv_heads":4}
1142	{"num_heads":8,"num_kv_heads":4}
1198	{"layers":9,"model_dim":512,"num_heads":8,"num_kv_heads":4}
1234	{"heads":8,"kv_heads":8}
1240	{"heads":4}
1245	{"kv_heads":4}
1250	{"kv_heads":8}
1258	{"heads":8,"kv_heads":4}
1264	{"kv_heads":4,"head_dim":64}
1273	{"kv_heads":8}
1311	{"heads":8,"kv_heads":4}
1352	{"heads":6,"kv_heads":2}
1387	{"num_heads":16,"num_kv_heads":8}
1396	{"kv_heads":4}
1440	{"attention_heads":8,"kv_heads":4}
1460	{"heads":8,"kv_heads":4}
1490	{"num_heads":4,"num_kv_heads":2}
1505	{"num_heads":8,"num_kv_heads":4}
1514	{"kv_heads":4}
1527	{"heads":4,"kv_heads":4}
1556	{"kv_heads":1}
1570	{"heads":8,"kv_heads":4}
1602	{"kv_heads":8}
1617	{"query_heads":8,"kv_heads":4}
1628	{"heads":8,"kv_heads":4}
1646	{"heads":8,"kv_heads":4}
1661	{"num_heads":8,"num_kv_heads":4}
1691	{"kv_heads":2}
1723	{"heads":8,"kv_heads":4}
1744	{"layers":11,"dimensions":512,"heads":8,"kv_heads":4}
1748	{"num_heads":8,"num_kv_heads":4}