Update ppo_model_utils.py (#10593)

* Update ppo_model_utils.py

* Update pp_model_utils.py

Author: ZHUI

paddlenlp/rl/models/pp_model_utils.py

@@ -38,9 +38,9 @@ def fwd_step_patch(func, output, self, *args, **kwargs):
     # Training patch
     if self.training and self.is_pipeline_last_stage():
         if getattr(self, "_step_losses", None):
-            self._step_losses.append(output.detach())
+            self._step_losses.append(output[0].detach())
         else:
-            self._step_losses = [output.detach()]
+            self._step_losses = [output[0].detach()]
 
 
 def make_wrapper(func, pre_patch=None, post_patch=None):

paddlenlp/rl/models/ppo_model_utils.py

@@ -351,7 +351,7 @@ def backward(ctx, grad_output: paddle.Tensor) -> paddle.Tensor:
 
 
 def entropy_from_logits(logits: paddle.Tensor, tensor_parallel_output=False):
-    return VocabParallelEntropy.apply(logits, tensor_parallel_output)
+    return VocabParallelEntropy.apply(logits.astype("float32"), tensor_parallel_output)
 
 
 @merge_fwd_labels