[Core] Unified quantization framework (#1764)
This commit is contained in:
1 parent
2a630313ce
commit
6b93459826
40 files changed
+3306
-605
No files matched your search
@@ -70,6 +70,12 @@ def parse_args():
|
||||
action="store_true",
|
||||
help="Enable diffusion pipeline profiler to display stage durations.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--quantization",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Quantization method (e.g. 'fp8').",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
@@ -119,6 +125,8 @@ def main():
|
||||
"enable_diffusion_pipeline_profiler": args.enable_diffusion_pipeline_profiler,
|
||||
}
|
||||
)
|
||||
if args.quantization:
|
||||
omni_kwargs["quantization_config"] = args.quantization
|
||||
|
||||
omni = Omni(model=model_name, **omni_kwargs)
|
||||
|
||||
|
||||
@@ -263,7 +263,7 @@ query_map = {
|
||||
|
||||
|
||||
def main(args):
|
||||
model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
model_name = args.model
|
||||
print("=" * 20, "\n", f"vllm version: {vllm.__version__}", "\n", "=" * 20)
|
||||
|
||||
# Get paths from args
|
||||
@@ -299,6 +299,7 @@ def main(args):
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
init_timeout=args.init_timeout,
|
||||
enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
|
||||
)
|
||||
|
||||
@@ -333,11 +334,14 @@ def main(args):
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
|
||||
sampling_params_list = [
|
||||
all_sampling_params = [
|
||||
thinker_sampling_params,
|
||||
talker_sampling_params, # code predictor is integrated into talker for Qwen3 Omni
|
||||
code2wav_sampling_params,
|
||||
]
|
||||
# Match sampling params to the number of configured stages
|
||||
num_stages = omni.num_stages
|
||||
sampling_params_list = all_sampling_params[:num_stages]
|
||||
|
||||
if args.txt_prompts is None:
|
||||
prompts = [query_result.inputs for _ in range(args.num_prompts)]
|
||||
@@ -415,6 +419,12 @@ def main(args):
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
help="Model name or path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--query-type",
|
||||
"-q",
|
||||
|
||||
Reference in new issue
Block a user