[Edge][CPU] Compiled post-step also covers the Identity talker->predictor projection (0.6B)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
3085f50160
commit
9502d3cef7
1 file changed
+8
-7
@@ -466,7 +466,7 @@ def _post_step_sample(
|
|||||||
hidden: torch.Tensor,
|
hidden: torch.Tensor,
|
||||||
head_w: torch.Tensor,
|
head_w: torch.Tensor,
|
||||||
embed_w: torch.Tensor,
|
embed_w: torch.Tensor,
|
||||||
proj_w: torch.Tensor,
|
proj_w: torch.Tensor | None,
|
||||||
proj_b: torch.Tensor | None,
|
proj_b: torch.Tensor | None,
|
||||||
u: torch.Tensor,
|
u: torch.Tensor,
|
||||||
inv_temperature: float,
|
inv_temperature: float,
|
||||||
@@ -487,8 +487,8 @@ def _post_step_sample(
|
|||||||
code = (scaled.float() - torch.log(-torch.log(u))).argmax(dim=-1, keepdim=True)
|
code = (scaled.float() - torch.log(-torch.log(u))).argmax(dim=-1, keepdim=True)
|
||||||
else:
|
else:
|
||||||
code = logits.argmax(dim=-1, keepdim=True)
|
code = logits.argmax(dim=-1, keepdim=True)
|
||||||
new_embed = F.embedding(code, embed_w) # [B, 1, H]
|
new_embed = F.embedding(code, embed_w).reshape(code.shape[0], -1) # [B, H]
|
||||||
proj_row = F.linear(new_embed.reshape(code.shape[0], -1), proj_w, proj_b)
|
proj_row = new_embed if proj_w is None else F.linear(new_embed, proj_w, proj_b)
|
||||||
return code, proj_row
|
return code, proj_row
|
||||||
|
|
||||||
|
|
||||||
@@ -1483,10 +1483,11 @@ class CodePredictorWrapper(nn.Module):
|
|||||||
and not stored_mode
|
and not stored_mode
|
||||||
and getattr(self, "_parity_ref", None) is None
|
and getattr(self, "_parity_ref", None) is None
|
||||||
and not self._wrapper_config.return_proj_buf
|
and not self._wrapper_config.return_proj_buf
|
||||||
and isinstance(projection, nn.Linear)
|
and isinstance(projection, (nn.Linear, nn.Identity))
|
||||||
):
|
):
|
||||||
head = lm_heads[step - 1]
|
head = lm_heads[step - 1]
|
||||||
proj = projection
|
proj_w = projection.weight if isinstance(projection, nn.Linear) else None
|
||||||
|
proj_b = projection.bias if isinstance(projection, nn.Linear) else None
|
||||||
u = torch.empty(bsz, head.weight.shape[0], dtype=torch.float32, device=device)
|
u = torch.empty(bsz, head.weight.shape[0], dtype=torch.float32, device=device)
|
||||||
row_generators = self._normalize_generators(sample_generator, bsz)
|
row_generators = self._normalize_generators(sample_generator, bsz)
|
||||||
if isinstance(row_generators, list):
|
if isinstance(row_generators, list):
|
||||||
@@ -1498,8 +1499,8 @@ class CodePredictorWrapper(nn.Module):
|
|||||||
hidden_step,
|
hidden_step,
|
||||||
head.weight,
|
head.weight,
|
||||||
codec_embeds[step - 1].weight,
|
codec_embeds[step - 1].weight,
|
||||||
proj.weight,
|
proj_w,
|
||||||
proj.bias,
|
proj_b,
|
||||||
u,
|
u,
|
||||||
float(sample_kwargs["inv_temperature"]),
|
float(sample_kwargs["inv_temperature"]),
|
||||||
int(sample_kwargs["top_k"]),
|
int(sample_kwargs["top_k"]),
|
||||||
|
|||||||
Reference in new issue
Block a user