add external dp launcher

Signed-off-by: whx-sjtu <2952154980@qq.com>

Author: whx-sjtu

examples/external_online_dp/launch_online_dp.py

@@ -0,0 +1,95 @@
+import argparse
+import multiprocessing
+import os
+import subprocess
+import sys
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dp-size",
+        type=int,
+        required=True,
+        help="Data parallel size."
+    )
+    parser.add_argument(
+        "--tp-size",
+        type=int,
+        default=1,
+        help="Tensor parallel size."
+    )
+    parser.add_argument(
+        "--dp-size-local",
+        type=int,
+        required=True,
+        help="Local data parallel size."
+    )
+    parser.add_argument(
+        "--dp-rank-start",
+        type=int,
+        default=0,
+        help="Starting rank for data parallel."
+    )
+    parser.add_argument(
+        "--dp-address",
+        type=str,
+        required=True,
+        help="IP address for data parallel master node."
+    )
+    parser.add_argument(
+        "--dp-rpc-port",
+        type=str,
+        required=True,
+        help="Port for data parallel master node."
+    )
+    parser.add_argument(
+        "--vllm-start-port",
+        type=int,
+        default=9000,
+        help="Starting port for the engine."
+    )
+    return parser.parse_args()
+
+args = parse_args()
+dp_size = args.dp_size
+tp_size = args.tp_size
+dp_size_local = args.dp_size_local
+dp_rank_start = args.dp_rank_start
+dp_address = args.dp_address
+dp_rpc_port = args.dp_rpc_port
+vllm_start_port = args.vllm_start_port
+
+def run_command(visiable_devices, dp_rank, vllm_engine_port):
+    command = [
+        "bash",
+        "./run_dp_template.sh",
+        visiable_devices,
+        str(vllm_engine_port),
+        str(dp_size),
+        str(dp_rank),
+        dp_address,
+        dp_rpc_port,
+        str(tp_size),
+    ]
+    subprocess.run(command, check=True)
+
+if __name__ == "__main__":
+    template_path = "./run_dp_template.sh"
+    if not os.path.exists(template_path):
+        print(f"Template file {template_path} does not exist.")
+        sys.exit(1)
+
+    processes = []
+    num_cards = dp_size_local * tp_size
+    for i in range(dp_size_local):
+        dp_rank = dp_rank_start + i
+        vllm_engine_port = vllm_start_port + i
+        visiable_devices = ",".join(str(x) for x in range(i * tp_size, (i + 1) * tp_size))
+        process = multiprocessing.Process(target=run_command,
+                                        args=(visiable_devices, dp_rank,
+                                                vllm_engine_port))
+        processes.append(process)
+        process.start()
+
+    for process in processes:
+        process.join()

examples/external_online_dp/run_dp_template.sh

@@ -0,0 +1,48 @@
+export HCCL_IF_IP=your_ip_here
+export GLOO_SOCKET_IFNAME=your_socket_ifname_here
+export TP_SOCKET_IFNAME=your_socket_ifname_here
+export HCCL_SOCKET_IFNAME=your_socket_ifname_here
+export DISAGGREGATED_PREFILL_RANK_TABLE_PATH=your_rank_table_path_here
+export VLLM_LOGGING_LEVEL="info"
+export OMP_PROC_BIND=false
+export OMP_NUM_THREADS=10
+export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
+export HCCL_DETERMINISTIC=True
+export HCCL_BUFFER_SIZE=1024
+export TASK_QUEUE_ENABLE=1
+# Spawn the process inside the vllm maybe cause the circular import issue, using fork here is necessary
+export VLLM_WORKER_MULTIPROC_METHOD="fork"
+
+export VLLM_USE_V1=1
+
+export ASCEND_RT_VISIBLE_DEVICES=$1
+
+vllm serve model_path \
+    --host 0.0.0.0 \
+    --port $2 \
+    --data-parallel-size $3 \
+    --data-parallel-rank $4 \
+    --data-parallel-address $5 \
+    --data-parallel-rpc-port $6 \
+    --tensor-parallel-size $7 \
+    --enable-expert-parallel \
+    --seed 1024 \
+    --served-model-name dsv3 \
+    --max-model-len 3500 \
+    --max-num-batched-tokens 3500 \
+    --max-num-seqs 28 \
+    --trust-remote-code \
+    --gpu-memory-utilization 0.9 \
+    --quantization ascend \
+    --speculative-config '{"num_speculative_tokens": 1, "method":"deepseek_mtp"}' \
+    --kv-transfer-config \
+    '{"kv_connector": "LLMDataDistCMgrConnector",
+      "kv_buffer_device": "npu",
+      "kv_role": "kv_consumer",
+      "kv_parallel_size": "1",
+      "kv_port": "20001",
+      "engine_id": "0",
+      "kv_connector_module_path": "vllm_ascend.distributed.llmdatadist_c_mgr_connector"
+    }' \
+    --additional-config \
+    '{"ascend_scheduler_config": {"enabled": true}, "torchair_graph_config":{"enabled":true,"enable_kv_nz":false, "enable_multistream_moe":false, "graph_batch_size":[28]}, "enable_weight_nz_layout":true}'