[Perf][Fish Speech] Free unused DAC codec components to save VRAM (#2430)

Signed-off-by: Sy03 <1370724210@qq.com>

vllm_omni/model_executor/models/fish_speech/dac_encoder.py

@@ -54,6 +54,9 @@ def _load_dac_codec(
     if "generator" in state_dict:
         state_dict = state_dict["generator"]
     codec.load_state_dict(state_dict, strict=False)
+    # Encoder path only uses encoder + quantizer.forward(); prune the
+    # decoder before moving to device to avoid unnecessary GPU allocation.
+    codec.decoder = None
     codec = codec.to(device=device, dtype=dtype)
     codec.eval()
 

vllm_omni/model_executor/models/fish_speech/fish_speech_dac_decoder.py

@@ -141,6 +141,13 @@ class FishSpeechDACDecoder(nn.Module):
         self._bake_weight_norm(codec)
         self._cache_attention_masks(codec)
 
+        # Decode path only uses quantizer.decode() + decoder; prune
+        # encode-only components before moving to device to avoid
+        # unnecessary GPU allocation.
+        codec.encoder = None
+        codec.quantizer.pre_module = None
+        codec.quantizer.downsample = None
+
         device = self.vllm_config.device_config.device
         codec = codec.to(device=device, dtype=torch.float32)
         codec.eval()