Merge branch 'main' into dev_accept_sim

Author: zyksir

.github/workflows/test.yaml

@@ -18,7 +18,7 @@ jobs:
         github.event.pull_request.draft == false
     runs-on: [self-hosted]
     container:
-      image: lmsysorg/sglang:dev
+      image: lmsysorg/sglang:v0.5.5 # we lock to this version to avoid repeated docker pull
       options: --gpus all --shm-size=2g --rm -v /dev/shm
     steps:
       - name: Checkout code
@@ -45,6 +45,7 @@ jobs:
         shell: bash
         run: |
           source sf/bin/activate
+          export PYTHONPATH=$PWD
           python -m unittest discover -s ./tests -p "test_*.py" -v
 
       - name: Save cache

configs/qwen2.5-7b-eagle3.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLMEagle3"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "llama",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064,
+  "draft_vocab_size": 16000
+}

examples/prepare_hidden_states.sh

@@ -0,0 +1,30 @@
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+ROOT_DIR=$(dirname $SCRIPT_DIR)
+
+export PYTHONPATH=$ROOT_DIR
+
+MODEL_PATH=Qwen/Qwen-2.5-7B-Instruct
+MODEL_NAME=Qwen-2.5-7B-Instruct
+DATASET_PATH=/datasets/all_2w.jsonl
+OUTPUT_ROOT_DIR=/specforge_output/$MODEL_NAME/2w
+NUM_SAMPLES=20000
+TP_SIZE=1
+MAX_LENGTH=16384
+DRAFT_CONFIG_PATH=./configs/qwen2.5-7b-eagle3.json
+
+OUTPUT_DIR=$OUTPUT_ROOT_DIR/outputs
+NPROC_PER_NODE=8
+
+
+torchrun \
+    --nproc_per_node $NPROC_PER_NODE \
+    scripts/prepare_hidden_states.py \
+    --target-model-path $MODEL_PATH \
+    --enable-aux-hidden-states \
+    --data-path $DATASET_PATH \
+    --chat-template qwen \
+    --max-length $MAX_LENGTH \
+    --tp-size $TP_SIZE \
+    --output-path $OUTPUT_ROOT_DIR/cache/hidden_states \
+    --num-samples $NUM_SAMPLES \
+    --batch-size 4

requirements-rocm.txt

@@ -0,0 +1,19 @@
+# Use the PyTorch ROCm wheel index (choose the stream that matches your system)
+--extra-index-url https://download.pytorch.org/whl/rocm6.3
+
+pre-commit
+torch==2.8.0+rocm6.3
+torchaudio==2.8.0+rocm6.3
+torchvision==0.23.0+rocm6.3
+transformers==4.57.1
+qwen-vl-utils==0.0.11
+datasets
+setuptools
+tqdm
+wandb
+psutil
+numpy
+accelerate
+pydantic
+sglang[all]==0.5.4
+openai-harmony