Automatic Prefix Caching - acc fix (#900)

Fix for accuracy while using APC --------- Co-authored-by: root <root@adobrzyniewicz-li4f-g2-mpijob-worker-0.adobrzyniewicz-li4f-g2-mpijob-worker.framework.svc.cluster.local>
HabanaAI · Mar 10, 2025 · 489a526 · 489a526
1 parent 88dc1de
commit 489a526
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/vllm/worker/hpu_model_runner.py b/vllm/worker/hpu_model_runner.py
@@ -279,7 +279,8 @@ def _compile_region(self, model, name, module):
     def _set_attn_bias(self, attn_metadata, batch_size, seq_len, device,
                        dtype):
         if (attn_metadata is None
-                or (self.prefill_use_fusedsdpa and self.is_causal)
+                or (self.prefill_use_fusedsdpa and self.is_causal
+                    and attn_metadata.block_list is None)
                 or not attn_metadata.is_prompt):
             return attn_metadata