task: xsum
checkpoints:
  generator: generator.pt
  tokenizer: tokenizer.pt
data:
  test: passages.pickle
  test_refs: reference_summaries.pickle
  validation: validation_cond512_le48.pickle
model:
  n_layer: 12
  n_head: 12
  bias: true
  dropout: 0.3
  n_embd: 768
  context_length: 560
  prefix_len: 512
  raw_token_prefix: true
  prefix_pad_token_id: 50257
  vocab_size:
  - 4096
  - 4096
  - 4096
  - 4096
  - 4096
  - 4096
  - 4096
  - 4096
  - 4096
  predict_level0: false
  attn_pattern: block_diagonal
  use_positional_encoding: true
  use_level_encoding: true
  use_rope: true
  rope_base: 100000
  use_qk_norm: true
  ctx_encoder:
    enabled: false
    n_layer: 4
    vocab_size: 50304
  use_flex_attention: false
  query_level0: true
  tie_heads_to_codebook: true
  tie_heads_learn_scale: true
tokenizer:
  pre_tokenizer:
    name: hf
    tokenizer: hf
    model_id: gpt2
    special_tokens:
      eos_token: <|endoftext|>
      pad_token: <|pad|>
  vqvae:
    n_layers: 6
    embed_dim: 384
    in_vocab_size: 50304
    context_length: 48
    compression_factor: 1
    dropout: 0.1
    pre_quant_groupnorm: 8
    use_qk_norm: true
    pre_quant_dropout: 0.25
    vqvae_vocab_size:
    - 4096
    - 4096
    - 4096
    - 4096
    - 4096
    - 4096
    - 4096
    - 4096
    - 4096
    checkpoint_path: tokenizer.pt
    vector_quantizer_config:
      clss: multiscale_residual_vector_quantizer
      decay: 0.99
      epsilon: 1.0e-05
      commitment_cost: 0.25
      learned_l1_sampling: true
      learned_l1_cascade: true
      learned_l1_cascade_base: 4
      learned_all_sampling: false
      quant_resi:
        enabled: true
        ratio: 0.5
        share_mode: 0
        learnable_ratio: false
      aux:
        fine_drop:
          prob: 0.5
          min_keep: 1
      levels:
        use_manual_levels: true
        manual_levels:
        - 4
        - 6
        - 10
        - 14
        - 20
        - 26
        - 32
        - 40
        - 48
      learned_l1_cascade_norm: true
      learned_l1_cascade_norm_final: true
generation:
  temperature: 1.0
  top_k: 1
  top_p: 0.0
  cfg_scale: 0.0
  frequency_penalty: 2.0
  frequency_deficit: 1.0
  adjacency_penalty: 5,4,0.5
  frequency_allowance: generator.pt:inference.token_unigram
eval:
  split: test
  n_docs: 11334
  batch_size: 32
  n_shards: 2
  metric: rouge