Force the selection operand of a GIMPLE COND_EXPR to be a register

This goes away with the selection operand allowed to be a GENERIC
tcc_comparison tree.  It keeps those for vectorizer pattern recog,
those are short lived and removing this instance is a bigger task.

The patch doesn't yet remove dead code and functionality, that's
left for a followup.  Instead the patch makes sure to produce
valid GIMPLE IL and continue to optimize COND_EXPRs where the
previous IL allowed and the new IL showed regressions in the testsuite.

2022-05-16  Richard Biener  <rguenther@suse.de>

	* gimple-expr.cc (is_gimple_condexpr): Equate to is_gimple_val.
	* gimplify.cc (gimplify_pure_cond_expr): Gimplify the condition
	as is_gimple_val.
	* gimple-fold.cc (valid_gimple_rhs_p): Simplify.
	* tree-cfg.cc (verify_gimple_assign_ternary): Likewise.
	* gimple-loop-interchange.cc (loop_cand::undo_simple_reduction):
	Build the condition of the COND_EXPR separately.
	* tree-ssa-loop-im.cc (move_computations_worker): Likewise.
	* tree-vect-generic.cc (expand_vector_condition): Likewise.
	* tree-vect-loop.cc (vect_create_epilog_for_reduction):
	Likewise.
	* vr-values.cc (simplify_using_ranges::simplify): Likewise.
	* tree-vect-patterns.cc: Add comment indicating we are
	building invalid COND_EXPRs and why.
	* omp-expand.cc (expand_omp_simd): Gimplify the condition
	to the COND_EXPR separately.
	(expand_omp_atomic_cas): Note part that should be unreachable
	now.
	* tree-ssa-forwprop.cc (forward_propagate_into_cond): Adjust
	condition for valid replacements.
	* tree-if-conv.cc (predicate_bbs): Simulate previous
	re-folding of the condition in folded COND_EXPRs which
	is necessary because of unfolded GIMPLE_CONDs in the IL
	as in for example gcc.dg/fold-bopcond-1.c.
	* gimple-range-gori.cc (gori_compute::condexpr_adjust):
	Handle that the comparison is now in the def stmt of
	the select operand.  Required by gcc.dg/pr104526.c.

	* gcc.dg/gimplefe-27.c: Adjust.
	* gcc.dg/gimplefe-45.c: Likewise.
	* gcc.dg/pr101145-2.c: Likewise.
	* gcc.dg/pr98211.c: Likewise.
	* gcc.dg/torture/pr89595.c: Likewise.
	* gcc.dg/tree-ssa/divide-7.c: Likewise.
	* gcc.dg/tree-ssa/ssa-lim-12.c: Likewise.

21 files changed, 104 insertions, 62 deletions

diff --git a/gcc/gimple-expr.cc b/gcc/gimple-expr.cc
index 5d10c24ed1b..09eac80ae1c 100644
--- a/gcc/gimple-expr.cc
+++ b/gcc/gimple-expr.cc
@@ -622,7 +622,7 @@ is_gimple_condexpr (tree t)
 {
   /* Always split out _Complex type compares since complex lowering
      doesn't handle this case.  */
-  return is_gimple_condexpr_1 (t, true, false);
+  return is_gimple_val (t);
 }
 
 /* Like is_gimple_condexpr, but does not allow T to trap.  */
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index 8555a2bca89..f61bc87da63 100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -418,9 +418,7 @@ valid_gimple_rhs_p (tree expr)
 	default:
 	  if (get_gimple_rhs_class (code) == GIMPLE_TERNARY_RHS)
 	    {
-	      if ((code == COND_EXPR
-		   ? !is_gimple_condexpr (TREE_OPERAND (expr, 0))
-		   : !is_gimple_val (TREE_OPERAND (expr, 0)))
+	      if (!is_gimple_val (TREE_OPERAND (expr, 0))
 		  || !is_gimple_val (TREE_OPERAND (expr, 1))
 		  || !is_gimple_val (TREE_OPERAND (expr, 2)))
 		return false;
diff --git a/gcc/gimple-loop-interchange.cc b/gcc/gimple-loop-interchange.cc
index 53adc02acc2..d5d690b2407 100644
--- a/gcc/gimple-loop-interchange.cc
+++ b/gcc/gimple-loop-interchange.cc
@@ -897,7 +897,9 @@ loop_cand::undo_simple_reduction (reduction_p re, bitmap dce_seeds)
       /* Init new_var to MEM_REF or CONST depending on if it is the first
 	 iteration.  */
       induction_p iv = m_inductions[0];
-      cond = fold_build2 (NE_EXPR, boolean_type_node, iv->var, iv->init_val);
+      cond = make_ssa_name (boolean_type_node);
+      stmt = gimple_build_assign (cond, NE_EXPR, iv->var, iv->init_val);
+      gimple_seq_add_stmt_without_update (&stmts, stmt);
       new_var = copy_ssa_name (re->var);
       stmt = gimple_build_assign (new_var, COND_EXPR, cond, tmp, re->init);
       gimple_seq_add_stmt_without_update (&stmts, stmt);
diff --git a/gcc/gimple-range-gori.cc b/gcc/gimple-range-gori.cc
index 772ccb88d55..3e15eb5192d 100644
--- a/gcc/gimple-range-gori.cc
+++ b/gcc/gimple-range-gori.cc
@@ -1319,17 +1319,21 @@ gori_compute::condexpr_adjust (irange &r1, irange &r2, gimple *, tree cond,
   tree ssa2 = gimple_range_ssa_p (op2);
   if (!ssa1 && !ssa2)
     return false;
-  if (!COMPARISON_CLASS_P (cond))
+  if (TREE_CODE (cond) != SSA_NAME)
     return false;
-  tree type = TREE_TYPE (TREE_OPERAND (cond, 0));
-  if (!range_compatible_p (type, TREE_TYPE (TREE_OPERAND (cond, 1))))
+  gassign *cond_def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (cond));
+  if (!cond_def
+      || TREE_CODE_CLASS (gimple_assign_rhs_code (cond_def)) != tcc_comparison)
     return false;
-  range_operator *hand = range_op_handler (TREE_CODE (cond), type);
+  tree type = TREE_TYPE (gimple_assign_rhs1 (cond_def));
+  if (!range_compatible_p (type, TREE_TYPE (gimple_assign_rhs2 (cond_def))))
+    return false;
+  range_operator *hand = range_op_handler (gimple_assign_rhs_code (cond_def), type);
   if (!hand)
     return false;
 
-  tree c1 = gimple_range_ssa_p (TREE_OPERAND (cond, 0));
-  tree c2 = gimple_range_ssa_p (TREE_OPERAND (cond, 1));
+  tree c1 = gimple_range_ssa_p (gimple_assign_rhs1 (cond_def));
+  tree c2 = gimple_range_ssa_p (gimple_assign_rhs2 (cond_def));
 
   // Only solve if there is one SSA name in the condition.
   if ((!c1 && !c2) || (c1 && c2))
@@ -1337,8 +1341,8 @@ gori_compute::condexpr_adjust (irange &r1, irange &r2, gimple *, tree cond,
 
   // Pick up the current values of each part of the condition.
   int_range_max cl, cr;
-  src.get_operand (cl, TREE_OPERAND (cond, 0));
-  src.get_operand (cr, TREE_OPERAND (cond, 1));
+  src.get_operand (cl, gimple_assign_rhs1 (cond_def));
+  src.get_operand (cr, gimple_assign_rhs2 (cond_def));
 
   tree cond_name = c1 ? c1 : c2;
   gimple *def_stmt = SSA_NAME_DEF_STMT (cond_name);
diff --git a/gcc/gimplify.cc b/gcc/gimplify.cc
index 260993be215..2b92b6394dc 100644
--- a/gcc/gimplify.cc
+++ b/gcc/gimplify.cc
@@ -4257,14 +4257,14 @@ gimplify_pure_cond_expr (tree *expr_p, gimple_seq *pre_p)
     TREE_SET_CODE (cond, TRUTH_AND_EXPR);
   else if (code == TRUTH_ORIF_EXPR)
     TREE_SET_CODE (cond, TRUTH_OR_EXPR);
-  ret = gimplify_expr (&cond, pre_p, NULL, is_gimple_condexpr, fb_rvalue);
+  ret = gimplify_expr (&cond, pre_p, NULL, is_gimple_val, fb_rvalue);
   COND_EXPR_COND (*expr_p) = cond;
 
   tret = gimplify_expr (&COND_EXPR_THEN (expr), pre_p, NULL,
-				   is_gimple_val, fb_rvalue);
+			is_gimple_val, fb_rvalue);
   ret = MIN (ret, tret);
   tret = gimplify_expr (&COND_EXPR_ELSE (expr), pre_p, NULL,
-				   is_gimple_val, fb_rvalue);
+			is_gimple_val, fb_rvalue);
 
   return MIN (ret, tret);
 }
diff --git a/gcc/omp-expand.cc b/gcc/omp-expand.cc
index 9fcc67a3448..5729a20397d 100644
--- a/gcc/omp-expand.cc
+++ b/gcc/omp-expand.cc
@@ -6613,9 +6613,9 @@ expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
       altn2 = create_tmp_var (TREE_TYPE (altv));
       expand_omp_build_assign (&gsi, altn2, t);
       tree t2 = fold_convert (TREE_TYPE (fd->loop.v), n2);
+      t2 = fold_build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t2);
       t2 = force_gimple_operand_gsi (&gsi, t2, true, NULL_TREE,
 				     true, GSI_SAME_STMT);
-      t2 = fold_build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t2);
       gassign *g = gimple_build_assign (altn2, COND_EXPR, t2, altn2,
 					build_zero_cst (TREE_TYPE (altv)));
       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
@@ -6989,10 +6989,10 @@ expand_omp_simd (struct omp_region *region, struct omp_for_data *fd)
 	      tree t2 = fold_convert (TREE_TYPE (fd->loops[i + 1].v),
 				      fd->loops[i + 1].m2
 				      ? n2v : fd->loops[i + 1].n2);
-	      t2 = force_gimple_operand_gsi (&gsi, t2, true, NULL_TREE,
-					     true, GSI_SAME_STMT);
 	      t2 = fold_build2 (fd->loops[i + 1].cond_code, boolean_type_node,
 				fd->loops[i + 1].v, t2);
+	      t2 = force_gimple_operand_gsi (&gsi, t2, true, NULL_TREE,
+					     true, GSI_SAME_STMT);
 	      gassign *g
 		= gimple_build_assign (altn2, COND_EXPR, t2, altn2,
 				       build_zero_cst (TREE_TYPE (altv)));
@@ -8978,6 +8978,7 @@ expand_omp_atomic_cas (basic_block load_bb, tree addr,
   tree cond_op1, cond_op2;
   if (cond_stmt)
     {
+      /* We should now always get a separate cond_stmt.  */
       if (!operand_equal_p (cond, gimple_assign_lhs (cond_stmt)))
 	return false;
       cond_op1 = gimple_assign_rhs1 (cond_stmt);
diff --git a/gcc/testsuite/gcc.dg/gimplefe-27.c b/gcc/testsuite/gcc.dg/gimplefe-27.c
index 604a2cc2fcc..0053222c867 100644
--- a/gcc/testsuite/gcc.dg/gimplefe-27.c
+++ b/gcc/testsuite/gcc.dg/gimplefe-27.c
@@ -4,6 +4,8 @@ int __GIMPLE ()
 p (int n)
 {
   int _2;
-  _2 = n_1(D) != 0 ? 2 : 0;
+  _Bool _3;
+  _3 = n_1(D) != 0;
+  _2 = _3 ? 2 : 0;
   return _2;
 }
diff --git a/gcc/testsuite/gcc.dg/gimplefe-45.c b/gcc/testsuite/gcc.dg/gimplefe-45.c
index b1d3cbb0205..8e1447ff3ae 100644
--- a/gcc/testsuite/gcc.dg/gimplefe-45.c
+++ b/gcc/testsuite/gcc.dg/gimplefe-45.c
@@ -10,10 +10,12 @@ p (int n)
   int *_2;
   int *_t;
   int *_t1;
+  _Bool _3;
   _t = (int*)8;
   _t1 = 0;
   n = n & 2;
-  _2 = n != 0 ? _t : _t1;
+  _3 = n != 0;
+  _2 = _3 ? _t : _t1;
   return _2;
 }
 
diff --git a/gcc/testsuite/gcc.dg/pr101145-2.c b/gcc/testsuite/gcc.dg/pr101145-2.c
index 6ecfeb2c0d5..dad192cfebf 100644
--- a/gcc/testsuite/gcc.dg/pr101145-2.c
+++ b/gcc/testsuite/gcc.dg/pr101145-2.c
@@ -12,4 +12,6 @@ unsigned foo(unsigned val, unsigned start)
   return cnt;
 }
 
-/* { dg-final { scan-tree-dump "cnt_\[0-9\] = start_\[0-9\]\\(D\\) >= val_\[0-9\]\\(D\\) \\? _\[0-9\] : 1;" "optimized" } } */
+/* Look for start >= val ? -start : 1  */
+/* { dg-final { scan-tree-dump " = start_\[0-9\]\\(D\\) >= val_\[0-9\]\\(D\\);" "optimized" } } */
+/* { dg-final { scan-tree-dump "cnt_\[0-9\] = _\[0-9\]+ \\? _\[0-9\]+ : 1;" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/pr98211.c b/gcc/testsuite/gcc.dg/pr98211.c
index cea371dcee7..8f14c8f98e0 100644
--- a/gcc/testsuite/gcc.dg/pr98211.c
+++ b/gcc/testsuite/gcc.dg/pr98211.c
@@ -13,15 +13,19 @@ test (int var_1, short int a, short int b, short int c, short int d)
   _Bool _28;
   short int _30;
   short int _32;
+  _Bool _29;
+  _Bool _31;
 
   __BB(2):
   _24 = test_var_3;
   tem_25 = _24 != 0;
   tem2_26 = var_1_11(D) != 0;
   _28 = tem_25 | tem2_26;
-  _30 = _28 !=  _Literal (_Bool) 0 ? a_16(D) : b_15(D);
+  _29 = _28 !=  _Literal (_Bool) 0;
+  _30 = _29 ? a_16(D) : b_15(D);
   arr_20[0u] = _30;
-  _32 = _28 != _Literal (_Bool) 0 ? c_19(D) : d_18(D);
+  _31 = _28 != _Literal (_Bool) 0;
+  _32 = _31 ? c_19(D) : d_18(D);
   arr_20[8u] = _32;
   arr_20[1u] = _30;
   arr_20[9u] = _32;
diff --git a/gcc/testsuite/gcc.dg/torture/pr89595.c b/gcc/testsuite/gcc.dg/torture/pr89595.c
index f45dc98c3f0..3a4bed9add8 100644
--- a/gcc/testsuite/gcc.dg/torture/pr89595.c
+++ b/gcc/testsuite/gcc.dg/torture/pr89595.c
@@ -5,6 +5,7 @@ int __attribute__((noipa))
 __GIMPLE(ssa,startwith("dom")) bar(int cond, int val)
 {
   int i;
+  _Bool _7;
 
 __BB(3):
   if (0 != 0)
@@ -20,7 +21,8 @@ __BB(2):
 
 __BB(4):
   i_6 = val_2(D);
-  i_1 = val_2(D) > 0 ? i_6 : 0;
+  _7 = val_2(D) > 0;
+  i_1 = _7 ? i_6 : 0;
   goto __BB5;
 
 __BB(5):
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/divide-7.c b/gcc/testsuite/gcc.dg/tree-ssa/divide-7.c
index c605515af56..c577ea3fbe1 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/divide-7.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/divide-7.c
@@ -6,4 +6,5 @@ int f(int x) {
 }
 
 /* { dg-final { scan-tree-dump-not "1 / x_\[0-9]\+\\\(D\\\);" "optimized" } } */
-/* { dg-final { scan-tree-dump " <= 2 \\? x_\[0-9]\+\\\(D\\\) : 0;" "optimized" } } */
+/* { dg-final { scan-tree-dump " <= 2;" "optimized" } } */
+/* { dg-final { scan-tree-dump " \\? x_\[0-9]\+\\\(D\\\) : 0;" "optimized" } } */
diff --git a/gcc/testsuite/gcc.dg/tree-ssa/ssa-lim-12.c b/gcc/testsuite/gcc.dg/tree-ssa/ssa-lim-12.c
index 016ff22f39f..c483951395b 100644
--- a/gcc/testsuite/gcc.dg/tree-ssa/ssa-lim-12.c
+++ b/gcc/testsuite/gcc.dg/tree-ssa/ssa-lim-12.c
@@ -23,4 +23,4 @@ void bar (int x, int z)
     }
 }
 
-/* { dg-final { scan-tree-dump-times "!= 0 ? " 2 "lim2" } } */
+/* { dg-final { scan-tree-dump-times " = _\[0-9\]+ ? " 2 "lim2" } } */
diff --git a/gcc/tree-cfg.cc b/gcc/tree-cfg.cc
index 19ba09facce..0a962dd5421 100644
--- a/gcc/tree-cfg.cc
+++ b/gcc/tree-cfg.cc
@@ -4240,8 +4240,7 @@ verify_gimple_assign_ternary (gassign *stmt)
       return true;
     }
 
-  if ((rhs_code == COND_EXPR
-       ? !is_gimple_condexpr (rhs1) : !is_gimple_val (rhs1))
+  if (!is_gimple_val (rhs1)
       || !is_gimple_val (rhs2)
       || !is_gimple_val (rhs3))
     {
@@ -4284,17 +4283,8 @@ verify_gimple_assign_ternary (gassign *stmt)
 	  debug_generic_expr (rhs1_type);
 	  return true;
 	}
-      if (!is_gimple_val (rhs1))
-	return true;
       /* Fallthrough.  */
     case COND_EXPR:
-      if (!is_gimple_val (rhs1)
-	  && (!is_gimple_condexpr (rhs1)
-	      || verify_gimple_comparison (TREE_TYPE (rhs1),
-					   TREE_OPERAND (rhs1, 0),
-					   TREE_OPERAND (rhs1, 1),
-					   TREE_CODE (rhs1))))
-	return true;
       if (!useless_type_conversion_p (lhs_type, rhs2_type)
 	  || !useless_type_conversion_p (lhs_type, rhs3_type))
 	{
diff --git a/gcc/tree-if-conv.cc b/gcc/tree-if-conv.cc
index 4531ca58bdf..5b884aae3d6 100644
--- a/gcc/tree-if-conv.cc
+++ b/gcc/tree-if-conv.cc
@@ -1298,10 +1298,31 @@ predicate_bbs (loop_p loop)
 	  tree c2;
 	  edge true_edge, false_edge;
 	  location_t loc = gimple_location (stmt);
-	  tree c = build2_loc (loc, gimple_cond_code (stmt),
-				    boolean_type_node,
-				    gimple_cond_lhs (stmt),
-				    gimple_cond_rhs (stmt));
+	  tree c;
+	  /* gcc.dg/fold-bopcond-1.c shows that despite all forwprop passes
+	     conditions can remain unfolded because of multiple uses so
+	     try to re-fold here, especially to get precision changing
+	     conversions sorted out.  Do not simply fold the stmt since
+	     this is analysis only.  When conditions were embedded in
+	     COND_EXPRs those were folded separately before folding the
+	     COND_EXPR but as they are now outside we have to make sure
+	     to fold them.  Do it here - another opportunity would be to
+	     fold predicates as they are inserted.  */
+	  gimple_match_op cexpr (gimple_match_cond::UNCOND,
+				 gimple_cond_code (stmt),
+				 boolean_type_node,
+				 gimple_cond_lhs (stmt),
+				 gimple_cond_rhs (stmt));
+	  if (cexpr.resimplify (NULL, follow_all_ssa_edges)
+	      && cexpr.code.is_tree_code ()
+	      && TREE_CODE_CLASS ((tree_code)cexpr.code) == tcc_comparison)
+	    c = build2_loc (loc, (tree_code)cexpr.code, boolean_type_node,
+			    cexpr.ops[0], cexpr.ops[1]);
+	  else
+	    c = build2_loc (loc, gimple_cond_code (stmt),
+			    boolean_type_node,
+			    gimple_cond_lhs (stmt),
+			    gimple_cond_rhs (stmt));
 
 	  /* Add new condition into destination's predicate list.  */
 	  extract_true_false_edges_from_block (gimple_bb (stmt),
diff --git a/gcc/tree-ssa-forwprop.cc b/gcc/tree-ssa-forwprop.cc
index 0917202b74c..b582529c404 100644
--- a/gcc/tree-ssa-forwprop.cc
+++ b/gcc/tree-ssa-forwprop.cc
@@ -610,7 +610,7 @@ forward_propagate_into_cond (gimple_stmt_iterator *gsi_p)
     }
 
   if (tmp
-      && is_gimple_condexpr (tmp))
+      && is_gimple_val (tmp))
     {
       if (dump_file)
 	{
diff --git a/gcc/tree-ssa-loop-im.cc b/gcc/tree-ssa-loop-im.cc
index 6d9316eed1f..bfd6126a4a5 100644
--- a/gcc/tree-ssa-loop-im.cc
+++ b/gcc/tree-ssa-loop-im.cc
@@ -1241,8 +1241,11 @@ move_computations_worker (basic_block bb)
 	     edges of COND.  */
 	  extract_true_false_args_from_phi (dom, stmt, &arg0, &arg1);
 	  gcc_assert (arg0 && arg1);
-	  t = build2 (gimple_cond_code (cond), boolean_type_node,
-		      gimple_cond_lhs (cond), gimple_cond_rhs (cond));
+	  t = make_ssa_name (boolean_type_node);
+	  new_stmt = gimple_build_assign (t, gimple_cond_code (cond),
+					  gimple_cond_lhs (cond),
+					  gimple_cond_rhs (cond));
+	  gsi_insert_on_edge (loop_preheader_edge (level), new_stmt);
 	  new_stmt = gimple_build_assign (gimple_phi_result (stmt),
 					  COND_EXPR, t, arg0, arg1);
 	  todo |= TODO_cleanup_cfg;
diff --git a/gcc/tree-vect-generic.cc b/gcc/tree-vect-generic.cc
index d99e3207fbe..f7de64cea40 100644
--- a/gcc/tree-vect-generic.cc
+++ b/gcc/tree-vect-generic.cc
@@ -1131,15 +1131,15 @@ expand_vector_condition (gimple_stmt_iterator *gsi, bitmap dce_ssa_names)
 				       comp_width, comp_index);
 	  tree aa2 = tree_vec_extract (gsi, comp_inner_type, a2,
 				       comp_width, comp_index);
-	  aa = build2 (code, cond_type, aa1, aa2);
+	  aa = gimplify_build2 (gsi, code, cond_type, aa1, aa2);
 	}
       else if (a_is_scalar_bitmask)
 	{
 	  wide_int w = wi::set_bit_in_zero (i, TYPE_PRECISION (TREE_TYPE (a)));
 	  result = gimplify_build2 (gsi, BIT_AND_EXPR, TREE_TYPE (a),
 				    a, wide_int_to_tree (TREE_TYPE (a), w));
-	  aa = build2 (NE_EXPR, boolean_type_node, result,
-		       build_zero_cst (TREE_TYPE (a)));
+	  aa = gimplify_build2 (gsi, NE_EXPR, boolean_type_node, result,
+				build_zero_cst (TREE_TYPE (a)));
 	}
       else
 	aa = tree_vec_extract (gsi, cond_type, a, comp_width, comp_index);
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index ab7dade1c74..f204b72a752 100644
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -5697,14 +5697,13 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
 						     old_idx_val);
 		  gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
 		}
+	      tree cond = make_ssa_name (boolean_type_node);
+	      epilog_stmt = gimple_build_assign (cond, GT_EXPR,
+						 idx_val, old_idx_val);
+	      gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
 	      tree new_val = make_ssa_name (data_eltype);
-	      epilog_stmt = gimple_build_assign (new_val,
-						 COND_EXPR,
-						 build2 (GT_EXPR,
-							 boolean_type_node,
-							 idx_val,
-							 old_idx_val),
-						 val, old_val);
+	      epilog_stmt = gimple_build_assign (new_val, COND_EXPR,
+						 cond, val, old_val);
 	      gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
 	      idx_val = new_idx_val;
 	      val = new_val;
@@ -5747,10 +5746,11 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
 	     values.  Check the result and if it is induc_val then replace
 	     with the original initial value, unless induc_val is
 	     the same as initial_def already.  */
-	  tree zcompare = build2 (EQ_EXPR, boolean_type_node, new_temp,
-				  induc_val);
+	  tree zcompare = make_ssa_name (boolean_type_node);
+	  epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR,
+					     new_temp, induc_val);
+	  gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
 	  tree initial_def = reduc_info->reduc_initial_values[0];
-
 	  tmp = make_ssa_name (new_scalar_dest);
 	  epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
 					     initial_def, new_temp);
@@ -6039,10 +6039,11 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo,
 	     values.  Check the result and if it is induc_val then replace
 	     with the original initial value, unless induc_val is
 	     the same as initial_def already.  */
-	  tree zcompare = build2 (EQ_EXPR, boolean_type_node, new_temp,
-				  induc_val);
+	  tree zcompare = make_ssa_name (boolean_type_node);
+	  epilog_stmt = gimple_build_assign (zcompare, EQ_EXPR, new_temp,
+					     induc_val);
+	  gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
 	  tree initial_def = reduc_info->reduc_initial_values[0];
-
 	  tree tmp = make_ssa_name (new_scalar_dest);
 	  epilog_stmt = gimple_build_assign (tmp, COND_EXPR, zcompare,
 					     initial_def, new_temp);
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 8c61eb965a6..ac49c1a26e1 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -50,6 +50,12 @@ along with GCC; see the file COPYING3.  If not see
 #include "vec-perm-indices.h"
 #include "gimple-range.h"
 
+
+/* TODO:  Note the vectorizer still builds COND_EXPRs with GENERIC compares
+   in the first operand.  Disentangling this is future work, the
+   IL is properly transfered to VEC_COND_EXPRs with separate compares.  */
+
+
 /* Return true if we have a useful VR_RANGE range for VAR, storing it
    in *MIN_VALUE and *MAX_VALUE if so.  Note the range in the dump files.  */
 
diff --git a/gcc/vr-values.cc b/gcc/vr-values.cc
index 2cc5084c15a..47faa4ff938 100644
--- a/gcc/vr-values.cc
+++ b/gcc/vr-values.cc
@@ -4292,7 +4292,10 @@ simplify_using_ranges::simplify (gimple_stmt_iterator *gsi)
 	     in divide by zero, new_rhs1 / new_rhs will be NULL_TREE.  */
 	  if (new_rhs1 && new_rhs2)
 	    {
-	      tree cond = build2 (EQ_EXPR, boolean_type_node, cmp_var, val1);
+	      tree cond = gimple_build (gsi, true, GSI_SAME_STMT,
+					UNKNOWN_LOCATION,
+					EQ_EXPR, boolean_type_node,
+					cmp_var, val1);
 	      gimple_assign_set_rhs_with_ops (gsi,
 					      COND_EXPR, cond,
 					      new_rhs1,