MLP3x

pr_number	parameters
63	{"hidden_size":1344,"multiplier":2.625}
65	{"multiplier":3,"hidden_dim":1536}
66	{"hidden":1536,"multiplier":3}
69	{"layers":9,"hidden_dim":1536,"vocab_size":1024,"dim":512,"gqa_heads":8,"kv_heads":4}
70	{"mlp_mult":3,"hidden_size":1536}
76	{"multiplier":3}
81	{"mlp_mult":3}
86	{"mlp_mult":3,"hidden":1536}
88	{"MLP_HIDDEN":1536}
89	{"hidden_dim":1536}
99	{"mlp_mult":3,"num_layers":9,"model_dim":512,"num_heads":8,"num_kv_heads":4,"tie_embeddings":1}
102	{"mlp_mult":3,"hidden_dim":1536}
107	{"hidden_size":1488}
108	{"layers":11,"hidden_dim":1536}
110	{"layers":9,"ffn_schedule":[768,960,1152,1344,1536,1728,1920,2112,2304]}
114	{"mlp_hidden":1536,"default_mlp_hidden":1024}
116	{"hidden_size":1536,"mlp_mult":3}
117	{"multiplier":3}
120	{"hidden_dim":1536,"layers":9,"model_dim":512,"num_heads":8,"num_kv_heads":4}
122	{"hidden_dim":1536,"multiplier":3}
123	{"multiplier":3,"hidden_size":1536}
128	{"mlp_mult":3,"hidden":1536}
135	{"hidden_dimension":1536}
137	{"hidden":1536,"mlp_mult":3}
139	{"mlp_multiplier":3,"hidden_dim":2304}
150	{"hidden":1536}
156	{"dimensions":1536}
160	{"mlp_mult":3}
162	{"hidden":1536}
164	{"hidden_size":1536}
172	{"mlp_mult":3,"model_dim":512,"layers":9,"heads":8,"kv_heads":4}
173	{"hidden_size":1536,"base_hidden_size":1024}
175	{"expansion":3,"hidden_dim":1536,"negative_slope":0.5}
178	{"hidden":1536}
180	{"hidden":1536}
181	{"mlp_mult":3}
187	{"multiplier":3}
190	{"mlp_hidden":1344}
191	{"mlp_mult":3}
192	{"multiplier":3}
194	{"multiplier":3}
196	{"mlp_mult":3}
198	{"hidden_size":1536}
201	{"hidden":1536}
206	{"hidden_size":1536}
207	—
208	{"layers":11}
212	{"hidden":1536}
215	{"mlp_mult":3}
218	{"layers":3}
219	{"hidden":1536}
222	{"mlp_mult":3,"hidden_size":1536,"num_layers":10,"model_dim":512,"num_heads":8,"num_kv_heads":4}
223	{"hidden":1536}
225	{"layers":11,"dimensions":512,"mlp_hidden":1024}
230	{"hidden_dim":1536,"multiplier":3}
232	{"layers":11,"hidden_dim":1536}
236	{"hidden":1536}
238	{"multiplier":3}
243	{"hidden":1536}
246	{"hidden":1536}
249	{"hidden":1536}
251	{"mlp_mult":3}
254	{"layers":11,"hidden_dim":1536,"heads":8,"kv_heads":4}
264	{"multiplier":3,"hidden_size":1536}
265	{"hidden_dim":1536}
267	{"hidden_size":1536}
273	{"multiplier":3}
274	{"hidden_size":1536}
278	—
281	{"layers":11,"model_dim":512,"heads":8,"kv_heads":4,"mlp_hidden":1536}
287	{"hidden_size":1536}
289	{"hidden":1536,"multiplier":3}
294	—
295	{"hidden_size":1536}
296	{"hidden":1536}
297	{"hidden":1536,"model_dim":512,"layers":9}
302	{"layers":3}
303	—
304	{"hidden_size":1536}
305	{"hidden":1536}
306	{"expansion":3,"hidden_dim":1536}
307	{"multiplier":3}
312	{"mlp_mult":3}
316	—
317	—
324	{"layers":10,"hidden":1536,"mlp_expansion":3}
326	{"layers":10,"hidden":1536}
327	{"expansion":3}
330	{"hidden_size":1536}
331	{"layers":10,"hidden":1536}
333	{"multiplier":2.75,"hidden_size":1408}
334	—
338	{"expansion":3}
339	—
351	{"multiplier":3}
352	{"mlp_multiplier":3}
359	{"layers":11,"width":512}
361	{"expansion_ratio":3}
362	{"mlp_multiplier":3,"hidden_dim":1536}
364	{"layers":10}
366	{"hidden_dim":1536}
368	{"layers":10,"dim":512,"mlp_multiplier":3}
369	{"expansion":3}
370	{"hidden":1536}
374	{"multiplier":3}
375	{"multiplier":3}
376	{"hidden":1536}
379	{"expansion":3}
383	{"expansion":3}
385	{"mlp_mult":3,"hidden_size":1536}
388	—
389	—
390	{"expansion":3}
398	{"hidden":1536}
400	{"multiplier":3}
401	{"multiplier":3}
403	—
406	—
410	{"layers":3}
414	{"expansion":3}
415	—
418	{"expansion":3}
422	{"hidden":1536}
424	—
433	{"expansion":3}
442	{"hidden":1536}
443	{"multiplier":3}
444	{"mlp_mult":3}
445	—
446	{"layers":7,"width":512,"hidden":1536,"attention_heads":8,"kv_heads":4}
447	{"multiplier":3,"hidden_dim":1536}
448	{"mlp_mult":3}
450	{"hidden":1536}
453	{"hidden_size":1536}
454	{"hidden_dim":1536}
455	{"expansion":3}
456	{"expansion":3}
458	{"layers":3}
461	{"expansion":3}
465	{"hidden":1536}
466	{"multiplier":3}
467	{"multiplier":3}
469	{"hidden":1728}
473	{"multiplier":3}
474	—
481	{"multiplier":3}
483	{"mlp_mult":3}
485	{"hidden_size":1536}
486	—
487	{"multiplier":3}
488	{"layers":11,"model_dim":512,"num_heads":8,"num_kv_heads":4,"mlp_mult":3}
489	—
499	{"mlp_multiplier":4,"hidden_size":2560}
508	{"expansion":3}
510	{"expansion_factor":3,"activation":"relu²"}
512	{"hidden_size":1536}
515	{"layers":10,"mlp_hidden":1536,"embedding_dim":512,"GQA_heads":"8/4"}
516	{"multiplier":3}
518	—
525	—
528	—
529	—
533	{"expansion":3}
546	{"hidden":1536,"baseline_hidden":1024}
547	{"mlp_hidden_units":1536,"expansion_factor":3}
548	{"layers":10,"d_model":512,"heads":8,"kv_heads":4}
549	{"layers":3}
550	{"expansion_factor":3}
555	—
560	{"layers":10}
563	{"expansion_factor":3,"hidden_dim":1536,"activation":"ReLU²"}
564	{"expansion_factor":3,"activation":"relu-squared"}
568	{"hidden_size":1536}
573	{"hidden_dim":1536}
580	{"MLP_MULT":3}
583	{"multiplier":3,"hidden_dim":1536}
585	{"multiplier":3.5,"hidden_dim":1792}
586	{"expansion":3}
589	{"layers":3}
593	{"layers":3}
595	{"layers":10,"d_model":512,"heads":8,"kv_heads":4,"mlp_multiplier":3}
596	{"expansion":3}
598	{"hidden":1536,"activation":"ReLU²"}
605	—
612	{"hidden_dim":1536}
614	—
618	{"mlp_mult":3}
619	{"MLP_MULT":3,"NUM_LAYERS":10}
622	{"hidden":1536}
633	{"hidden_dim":1536}
642	—
644	{"expansion_factor":3,"hidden_dim":1536,"activation":"ReLU²"}
645	—
659	—
666	{"hidden":2304}
671	{"hidden_units":1536}
678	{"mlp_mult":3,"hidden":1536}
681	{"activation":"LeakyReLU(0.5)^2"}
682	{"mlp_mult":3}
684	—
685	—
688	{"activation":"LeakyReLU(0.5)^2"}
690	{"layers":3}
692	{"layers":3}
693	{"layers":3,"activation":"LeakyReLU(0.5)^2"}
698	{"multiplier":3}
700	{"multiplier":3.5}
705	{"hidden_multiplier":3,"hidden_dim":1536}
709	{"hidden_size":1536}
710	{"expansion":3}
713	{"layers":10,"dim":512}
714	—
715	{"multiplier":3}
728	{"layers":11}
730	{"mlp_mult":4}
734	{"multiplier":3}
736	{"layers":9,"mlp_multiplier":2}
738	{"layers":11,"dimensions":512}
741	—
744	—
752	{"activation":"LeakyReLU(0.5)^2"}
754	{"hidden_dim":1536}
755	{"mlp_mult":3,"hidden":1152}
757	—
760	{"multiplier":3}
761	{"multiplier":3}
764	{"layers":11,"dimensions":512,"gqa":"8/4"}
768	—
770	—
771	{"expansion":3}
774	{"hidden_size_multiplier":3}
777	{"activation":"LeakyReLU(0.5)^2"}
778	{"multiplier":3}
779	—
786	{"layers":3}
791	—
794	{"layers":3}
795	—
796	{"mlp_multiplier":3}
797	—
798	{"mlp_blocks":3}
799	{"mlp_mult":3}
805	{"mlp_multiplier":3}
806	—
807	—
808	—
809	{"multiplier":3}
810	{"hidden_size":1536}
811	—
827	{"multiplier":2}
828	{"layers":10,"d_model":512}
836	{"expansion":3}
838	{"hidden_dim":1536}
840	{"multiplier":3}
841	{"mlp_layers":3}
843	{"multiplier":3}
849	{"hidden":1792}
850	{"expansion":3,"hidden":1536}
851	{"multiplier":3.5}
854	{"layers":9,"model_dim":512,"heads":8,"kv_heads":4,"ffn_hidden_dim":1536}
874	{"mlp_multiplier":3}
880	{"multiplier":3.5}
891	—
892	—
896	{"multiplier":3}
900	—
907	{"mlp_dim":1536}
915	{"layers":3}
924	—
925	{"multiplier":3}
926	{"expansion":3}
929	{"layers":9,"hidden_size":1536,"mlp_mult":2}
933	{"hidden_size":768}
940	—
941	{"multiplier":3.5}
945	{"multiplier":3.5}
947	{"mlp_mult":3.2}
953	{"expansion":3.5}
967	{"expansion":3.5}
972	—
978	—
979	{"expansion":3}
982	{"hidden_multiplier":3}
988	{"multiplier":2.5}
989	{"multiplier":3,"hidden_dim":1536}
993	{"multiplier":3,"activation":"ReLU²"}
995	{"multiplier":3.5}
996	—
997	{"hidden":1536}
999	—
1002	—
1005	—
1007	{"expansion":3}
1009	{"layers":3}
1025	{"multiplier":3}
1026	—
1048	{"hidden":1536}
1051	{"negative_slope":0.5}
1059	{"mlp_mult":3}
1062	—
1068	{"mlp_mult":3}
1071	{"hidden_dim":1536}
1080	{"multiplier":3}
1085	{"expansion_factor":3}
1086	{"expansion":3}
1088	{"layers":3}
1095	{"multiplier":3}
1105	{"multiplier":3}
1106	—
1108	{"expansion":3}
1112	{"expansion":3}
1125	{"multiplier":3}
1127	{"hidden_size":1536}
1130	—
1139	{"hidden_dim":1920}
1141	{"mlp_multiplier":3}
1142	{"multiplier":3}
1144	{"multiplier":3}
1148	—
1167	{"mlp_mult":3}
1170	{"multiplier":3}
1174	{"num_layers":5,"model_dim":512,"mlp_mult":4,"num_heads":8,"num_kv_heads":4,"bigram_vocab_size":4096,"bigram_dim":1024}
1183	—
1185	—
1186	{"multiplier":3.25}
1190	{"layers":10,"mlp_expansion":3}
1202	{"multiplier":3}
1209	—
1216	{"multiplier":3,"activation":"LeakyReLU"}
1218	{"mlp_mult":4}
1226	{"hidden":1536}
1227	{"multiplier":2}
1228	—
1240	—
1254	{"expansion":3}
1256	—
1276	{"activation":"LeakyReLU","activation_slope":0.5}
1280	—
1298	{"activation":"LeakyReLU²"}
1312	{"multiplier":3}
1313	{"layers":3}
1321	{"layers":3}
1337	{"multiplier":3}
1359	{"multiplier":3}
1361	{"activation":"LeakyReLU²"}
1366	—
1368	{"hidden":1536}
1378	{"layers":11,"mlp_multiplier":3,"hidden_size":1536}
1379	{"multiplier":3}
1411	{"mlp_layers":3,"activation":"ReLU²"}
1417	{"expansion":3}
1422	{"multiplier":3}
1431	{"multiplier":3.5,"hidden_dim":1792}
1436	—
1444	—
1446	{"width":1536}
1453	{"multiplier":3,"hidden_dim":2640}
1463	{"hidden":1536}
1473	{"width_multiplier":3}
1479	{"activation":"LeakyReLU","multiplier":3}
1488	—
1492	{"multiplier":4}
1505	{"mlp_multiplier":3,"hidden_dim":1536}
1508	{"mlp_multiplier":3,"activation":"LeakyReLU"}
1527	{"multiplier":3}
1536	{"expansion":4}
1556	{"multiplier":3}
1557	{"layers":11,"dimensions":512,"heads":8,"kv_heads":4,"mlp_multiplier":4}
1559	{"multiplier":3}
1561	{"hidden_dim":2048}
1568	{"expansion":3}
1577	{"mlp_mult":1.9}
1581	{"mlp_mult":3}
1617	{"multiplier":3}
1626	{"hidden_multiplier":4}
1630	{"multiplier":3}
1647	{"multiplier":4}
1649	—
1666	{"multiplier":3}
1715	{"multiplier":4}
1722	{"hidden_multiplier":3}
1751	—