| encoder: | |
| name: null | |
| max_length: 512 | |
| dtype: bfloat16 | |
| trust_remote_code: false | |
| decoder: | |
| d_model: 128 | |
| n_layer: 5 | |
| n_head: 4 | |
| d_ff: 512 | |
| max_seq_len: 128 | |
| prefix_len: 4 | |
| dropout: 0.0 | |
| tie_embeddings: true | |
| cross_attention: false | |
| enc_memory_len: 128 | |
| pos_encoding: learned | |
| mlp_type: gelu | |
| copy_head: false | |
| data: | |
| hf_name: parquet | |
| data_files: /workspace-SR008.nfs2/nsushko/nsushko/polygon/tinypython/build/07b_holdout.parquet | |
| split: train | |
| val_fraction: 0.02 | |
| max_identifiers: 26 | |
| max_tokens: 128 | |
| instruction_field: instruction | |
| code_field: response | |
| stdlib_whitelist_path: null | |
| num_proc: 8 | |
| cache_dir: .cache/tiny_decoder/tpy07b_v1 | |
| drop_stub_bodies: false | |
| pack: false | |
| pack_len: 512 | |
| vocab: | |
| path: configs/vocab_98.yaml | |
| builtin_aliases_path: configs/builtin_aliases.yaml | |
| attr_aliases_path: configs/attr_aliases.yaml | |
| tokenizer: | |
| kind: t98 | |
| bpe_path: null | |
| train: | |
| batch_size: 128 | |
| epochs: 3 | |
| lr: 0.0003 | |
| warmup_frac: 0.05 | |
| weight_decay: 0.1 | |
| grad_clip: 1.0 | |
| early_stop_patience: 5 | |
| seed: 42 | |
| log_every: 50 | |
| eval_every_epochs: 1 | |
| wandb_project: null | |
| tensorboard_dir: null | |
| weighted_sampling: false | |
| init_from: null | |
| resume_from: null | |
| pretrain_sampling: short | |
| amp_dtype: bf16 | |
| compile: false | |
| bench_every_epochs: 0 | |
| bench_benchmark: both | |
| bench_max_problems: null | |
| bench_num_samples: 5 | |
| bench_temperature: 0.8 | |
| bench_pass_k: 1,5 | |
| bench_timeout: 3.0 | |
| bench_num_workers: 16 | |
| bench_max_new_tokens: null | |
| eval: | |
| split: val | |
| max_samples: 500 | |
| compute_exact_match: true | |
| compute_syntactic_validity: true | |
| run: | |
| dir: /workspace-SR008.nfs2/nsushko/nsushko/polygon/tiny_decoder/runs/v1_rerun/decoder_only | |
| save_every_epochs: 1 | |