x265: Fix a deadlock when failing to create the x265enc.

The GST_ELEMENT_ERROR will call the gst_object_get_path_string and
use gst_object_get_parent to get the full object path name, which
needs to lock the object. But we are already in a locked context and
so this will cause a deadlock, the pipeline can not exit normally.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2451>

1 files changed, 5 insertions, 2 deletions

diff --git a/ext/x265/gstx265enc.c b/ext/x265/gstx265enc.c
index 148a195fb..7e7e723ab 100644
--- a/ext/x265/gstx265enc.c
+++ b/ext/x265/gstx265enc.c
@@ -986,8 +986,7 @@ gst_x265_enc_init_encoder_locked (GstX265Enc * encoder)
 
   encoder->x265enc = encoder->api->encoder_open (&encoder->x265param);
   if (!encoder->x265enc) {
-    GST_ELEMENT_ERROR (encoder, STREAM, ENCODE,
-        ("Can not initialize x265 encoder."), (NULL));
+    GST_ERROR_OBJECT (encoder, "Can not open x265 encoder.");
     return FALSE;
   }
 
@@ -1012,6 +1011,10 @@ gst_x265_enc_init_encoder (GstX265Enc * encoder)
   result = gst_x265_enc_init_encoder_locked (encoder);
   GST_OBJECT_UNLOCK (encoder);
 
+  if (!result)
+    GST_ELEMENT_ERROR (encoder, STREAM, ENCODE,
+        ("Can not initialize x265 encoder."), (NULL));
+
   return result;
 }