diff --git a/FlagEmbedding/abc/finetune/embedder/AbsModeling.py b/FlagEmbedding/abc/finetune/embedder/AbsModeling.py
index 3e9e32cb3..38f87222b 100644
--- a/FlagEmbedding/abc/finetune/embedder/AbsModeling.py
+++ b/FlagEmbedding/abc/finetune/embedder/AbsModeling.py
@@ -226,7 +226,7 @@ def _compute_cross_device_neg_loss(self, q_reps, p_reps, teacher_targets=None, c
 
                 cross_idxs = torch.arange(cross_q_reps.size(0), device=cross_q_reps.device, dtype=torch.long)
                 cross_targets = cross_idxs * group_size # (world_size * batch_size)
-                loss += self.compute_loss(cross_scores, cross_targets)
+                loss += self.world_size * self.compute_loss(cross_scores, cross_targets)
             elif self.kd_loss_type == "m3_kd_loss":
                 cross_teacher_targets = self._dist_gather_tensor(teacher_targets)   # (world_size * batch_size, group_size)