x265: Fix a deadlock when failing to create the x265enc.

The GST_ELEMENT_ERROR will call the gst_object_get_path_string and
use gst_object_get_parent to get the full object path name, which
needs to lock the object. But we are already in a locked context and
so this will cause a deadlock, the pipeline can not exit normally.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2460>

1 files changed, 5 insertions, 2 deletions

diff --git a/ext/x265/gstx265enc.c b/ext/x265/gstx265enc.c
index d0dc3f761..430f11c98 100644
--- a/ext/x265/gstx265enc.c
+++ b/ext/x265/gstx265enc.c
@@ -980,8 +980,7 @@ gst_x265_enc_init_encoder_locked (GstX265Enc * encoder)
 
   encoder->x265enc = encoder->api->encoder_open (&encoder->x265param);
   if (!encoder->x265enc) {
-    GST_ELEMENT_ERROR (encoder, STREAM, ENCODE,
-        ("Can not initialize x265 encoder."), (NULL));
+    GST_ERROR_OBJECT (encoder, "Can not open x265 encoder.");
     return FALSE;
   }
 
@@ -1006,6 +1005,10 @@ gst_x265_enc_init_encoder (GstX265Enc * encoder)
   result = gst_x265_enc_init_encoder_locked (encoder);
   GST_OBJECT_UNLOCK (encoder);
 
+  if (!result)
+    GST_ELEMENT_ERROR (encoder, STREAM, ENCODE,
+        ("Can not initialize x265 encoder."), (NULL));
+
   return result;
 }