[Core] Unified quantization framework (#1764)

This commit is contained in:
SYLAR authored and GitHub committed 2026-03-24 18:56:55 +08:00
1 parent 2a630313ce
commit 6b93459826
40 files changed
+3306 -605

No files matched your search

@@ -70,6 +70,12 @@ def parse_args():
action="store_true",
help="Enable diffusion pipeline profiler to display stage durations.",
)
parser.add_argument(
"--quantization",
type=str,
default=None,
help="Quantization method (e.g. 'fp8').",
)
args = parser.parse_args()
return args
@@ -119,6 +125,8 @@ def main():
"enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
}
)
if args.quantization:
omni_kwargs["quantization_config"] = args.quantization
omni = Omni(model=model_name, **omni_kwargs)
@@ -263,7 +263,7 @@ query_map = {
def main(args):
model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
model_name = args.model
print("=" * 20, "\n", f"vllm version: {vllm.__version__}", "\n", "=" * 20)
# Get paths from args
@@ -299,6 +299,7 @@ def main(args):
stage_configs_path=args.stage_configs_path,
log_stats=args.log_stats,
stage_init_timeout=args.stage_init_timeout,
init_timeout=args.init_timeout,
enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
)
@@ -333,11 +334,14 @@ def main(args):
repetition_penalty=1.1,
)
sampling_params_list = [
all_sampling_params = [
thinker_sampling_params,
talker_sampling_params, # code predictor is integrated into talker for Qwen3 Omni
code2wav_sampling_params,
]
# Match sampling params to the number of configured stages
num_stages = omni.num_stages
sampling_params_list = all_sampling_params[:num_stages]
if args.txt_prompts is None:
prompts = [query_result.inputs for _ in range(args.num_prompts)]
@@ -415,6 +419,12 @@ def main(args):
def parse_args():
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
parser.add_argument(
"--model",
type=str,
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
help="Model name or path.",
)
parser.add_argument(
"--query-type",
"-q",