config.json

{
  "data": {
    "channels": 34,
    "text_vocab_size": 49280,
    "audio_vocab_size": 2050,
    "action_vocab_size": 2,
    "text_pad_token_id": 3,
    "text_new_word_token_id": 2,
    "text_zero_token_id": 7,
    "audio_pad_token_id": 2049,
    "audio_bos_token_id": 2048,
    "action_pad_token_id": 0,
    "action_new_word_token_id": 1,
    "delay_pattern": [
      16, 18, 18, 18, 18, 18, 18, 18,
      18, 18, 18, 18, 18, 18, 18, 18,
      18, 18, 18, 18, 18, 18, 18, 18,
      18, 18, 18, 18, 18, 18, 18, 18
    ],
    "first_word_min_start": 3,
    "max_pad": 8,
    "second_stream_ahead": 2
  },
  "model": {
    "decoder": {
      "n_layer": 28,
      "n_embd": 2048,
      "n_hidden": 6144,
      "gqa_query_heads": 16,
      "kv_heads": 8,
      "gqa_head_dim": 128
    },
    "depformer": {
      "n_layer": 4,
      "n_embd": 1024,
      "n_hidden": 3072,
      "gqa_query_heads": 8,
      "kv_heads": 8,
      "gqa_head_dim": 128,
      "apply_rope": true,
      "text_embedding": false
    },
    "linear": {
      "mlp_activations": [
        "silu",
        "linear"
      ]
    },
    "dropout": 0.0,
    "rope_min_timescale": 1,
    "rope_max_timescale": 10000.0,
    "normalization_layer_epsilon": 0.000001
  },
  "runtime": {
    "weights_schedule": [
      0, 0,
      1, 1, 1, 1,
      2, 2, 2, 2, 2, 2, 2, 2,
      3, 3, 3, 3, 3, 3, 3, 3,
      4, 4, 4, 4, 4, 4, 4, 4, 4
    ],
    "max_context_steps": 1500
  }
}