From 8dce85118ce6903bb3293e1b99225a73049ce5df Mon Sep 17 00:00:00 2001
From: Richard Guo <guofenglinux@gmail.com>
Date: Mon, 17 Aug 2026 10:01:36 +0900
Subject: [PATCH v3] Fix CPU cost of right-semi and right-anti hash joins

final_cost_hashjoin() assumed that the rows a hash join produces come
from the outer side.  That does not hold for JOIN_RIGHT_SEMI and
JOIN_RIGHT_ANTI, which produce rows from the inner side: matched inner
rows for the one, unmatched inner rows for the other.  hashjointuples,
which carries both the cpu_tuple_cost charge and the cost of the join
clauses not used for hashing, was therefore counting the wrong rows,
and could be far off in either direction.

Compute hashjointuples from the inner side for these two join types,
mirroring what JOIN_SEMI and JOIN_ANTI already do with the outer side.
The fraction we need is semifactors.outer_match_frac: it is derived
from the SpecialJoinInfo, so despite its name it always describes the
semijoin's left-hand side, which is the inner side here.  Compute the
semijoin factors for these join types in all cases; previously that
happened only when the inner side was provably unique.

The same mix-up affects outer_matched_rows when the inner side is
known unique, where the outer row count was multiplied by the inner
side's match fraction.  A unique inner side means each outer row has
at most one match, so use the number of matching pairs instead.

Nestloop and mergejoin need no equivalent fix: neither supports
JOIN_RIGHT_SEMI, nestloop doesn't support JOIN_RIGHT_ANTI either, and
final_cost_mergejoin() takes its count from approx_tuple_count(),
which multiplies the two input sizes together and so gives the same
answer whichever side is on the outside.
---
 src/backend/optimizer/path/costsize.c         | 76 ++++++++++++++-----
 src/backend/optimizer/path/joinpath.c         |  9 ++-
 src/include/nodes/pathnodes.h                 | 32 ++++----
 src/test/regress/expected/join.out            | 76 ++++++++++++++++---
 src/test/regress/expected/opr_sanity.out      | 14 ++--
 src/test/regress/expected/select_parallel.out | 34 ++++-----
 src/test/regress/sql/join.sql                 | 32 ++++++++
 7 files changed, 204 insertions(+), 69 deletions(-)

diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index fd794c946ab..91d3250154f 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -4463,6 +4463,7 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path,
 	Cost		cpu_per_tuple;
 	QualCost	hash_qual_cost;
 	QualCost	qp_qual_cost;
+	double		outer_matched_rows = 0;
 	double		hashjointuples;
 	double		virtualbuckets;
 	Selectivity innerbucketsize;
@@ -4611,7 +4612,6 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path,
 		path->jpath.jointype == JOIN_ANTI ||
 		extra->inner_unique)
 	{
-		double		outer_matched_rows;
 		Selectivity inner_scan_frac;
 
 		/*
@@ -4625,9 +4625,29 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path,
 		 * 2.0 to that fraction.  (If we used a larger fuzz factor, we'd have
 		 * to clamp inner_scan_frac to at most 1.0; but since match_count is
 		 * at least 1, no such clamp is needed now.)
+		 *
+		 * For RIGHT_SEMI or RIGHT_ANTI, we cannot compute outer_matched_rows
+		 * from the semifactors: outer_match_frac describes the semijoin's
+		 * LHS, which is the inner rel in these orientations.  Instead, count
+		 * the matching pairs with approx_tuple_count().  These join types
+		 * reach here only when the innerrel is known unique, so each outer
+		 * row matches at most one inner row and the number of pairs equals
+		 * the number of matched outer rows.  Uniqueness also fixes
+		 * match_count at 1, making inner_scan_frac 1.0.
 		 */
-		outer_matched_rows = rint(outer_path_rows * extra->semifactors.outer_match_frac);
-		inner_scan_frac = 2.0 / (extra->semifactors.match_count + 1.0);
+		if (path->jpath.jointype == JOIN_RIGHT_SEMI ||
+			path->jpath.jointype == JOIN_RIGHT_ANTI)
+		{
+			outer_matched_rows = Min(approx_tuple_count(root, &path->jpath,
+														hashclauses),
+									 outer_path_rows);
+			inner_scan_frac = 1.0;
+		}
+		else
+		{
+			outer_matched_rows = rint(outer_path_rows * extra->semifactors.outer_match_frac);
+			inner_scan_frac = 2.0 / (extra->semifactors.match_count + 1.0);
+		}
 
 		startup_cost += hash_qual_cost.startup;
 		run_cost += hash_qual_cost.per_tuple * outer_matched_rows *
@@ -4649,12 +4669,6 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path,
 		run_cost += hash_qual_cost.per_tuple *
 			(outer_path_rows - outer_matched_rows) *
 			clamp_row_est(inner_path_rows / virtualbuckets) * 0.05;
-
-		/* Get # of tuples that will pass the basic join */
-		if (path->jpath.jointype == JOIN_ANTI)
-			hashjointuples = outer_path_rows - outer_matched_rows;
-		else
-			hashjointuples = outer_matched_rows;
 	}
 	else
 	{
@@ -4671,14 +4685,34 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path,
 		startup_cost += hash_qual_cost.startup;
 		run_cost += hash_qual_cost.per_tuple * outer_path_rows *
 			clamp_row_est(inner_path_rows * innerbucketsize) * 0.5;
+	}
 
-		/*
-		 * Get approx # tuples passing the hashquals.  We use
-		 * approx_tuple_count here because we need an estimate done with
-		 * JOIN_INNER semantics.
-		 */
+	/*
+	 * Get # of tuples that will pass the basic join.
+	 *
+	 * A RIGHT_SEMI or RIGHT_ANTI join produces rows from the inner side: one
+	 * for each inner row that has a match, or that lacks one.  The fraction
+	 * we need is outer_match_frac, which always describes the semijoin's LHS,
+	 * ie, the inner side for these two join types.
+	 *
+	 * Everything else produces rows from the outer side.  For SEMI and
+	 * inner_unique joins that is the matched outer rows, and for ANTI the
+	 * unmatched ones, both available from outer_matched_rows computed above.
+	 * For plain joins, use approx_tuple_count(), which gives an estimate done
+	 * with JOIN_INNER semantics.
+	 */
+	if (path->jpath.jointype == JOIN_RIGHT_SEMI)
+		hashjointuples = clamp_row_est(inner_path_rows *
+									   extra->semifactors.outer_match_frac);
+	else if (path->jpath.jointype == JOIN_RIGHT_ANTI)
+		hashjointuples = clamp_row_est(inner_path_rows *
+									   (1.0 - extra->semifactors.outer_match_frac));
+	else if (path->jpath.jointype == JOIN_ANTI)
+		hashjointuples = outer_path_rows - outer_matched_rows;
+	else if (path->jpath.jointype == JOIN_SEMI || extra->inner_unique)
+		hashjointuples = outer_matched_rows;
+	else
 		hashjointuples = approx_tuple_count(root, &path->jpath, hashclauses);
-	}
 
 	/*
 	 * For each tuple that gets through the hashjoin proper, we charge
@@ -5281,11 +5315,17 @@ get_restriction_qual_cost(PlannerInfo *root, RelOptInfo *baserel,
  *	joinrel: join relation under consideration
  *	outerrel: outer relation under consideration
  *	innerrel: inner relation under consideration
- *	jointype: if not JOIN_SEMI or JOIN_ANTI, we assume it's inner_unique
+ *	jointype: if not JOIN_SEMI, JOIN_ANTI, JOIN_RIGHT_SEMI or JOIN_RIGHT_ANTI,
+ *			  we assume it's inner_unique
  *	sjinfo: SpecialJoinInfo relevant to this join
  *	restrictlist: join quals
  * Output parameters:
  *	*semifactors is filled in (see pathnodes.h for field definitions)
+ *
+ * Note that outer_match_frac is derived from sjinfo and so always describes
+ * the semijoin's LHS, whichever side is physically outer.  match_count, by
+ * contrast, is computed from the physical inner rel, and so is not meaningful
+ * for JOIN_RIGHT_SEMI or JOIN_RIGHT_ANTI.
  */
 void
 compute_semi_anti_join_factors(PlannerInfo *root,
@@ -5331,7 +5371,9 @@ compute_semi_anti_join_factors(PlannerInfo *root,
 	jselec = clauselist_selectivity(root,
 									joinquals,
 									0,
-									(jointype == JOIN_ANTI) ? JOIN_ANTI : JOIN_SEMI,
+									(jointype == JOIN_ANTI ||
+									 jointype == JOIN_RIGHT_ANTI) ?
+									JOIN_ANTI : JOIN_SEMI,
 									sjinfo);
 
 	/*
diff --git a/src/backend/optimizer/path/joinpath.c b/src/backend/optimizer/path/joinpath.c
index 713283a73aa..dfd08e7aeb1 100644
--- a/src/backend/optimizer/path/joinpath.c
+++ b/src/backend/optimizer/path/joinpath.c
@@ -249,10 +249,13 @@ add_paths_to_joinrel(PlannerInfo *root,
 														  &mergejoin_allowed);
 
 	/*
-	 * If it's SEMI, ANTI, or inner_unique join, compute correction factors
-	 * for cost estimation.  These will be the same for all paths.
+	 * If it's SEMI, ANTI, RIGHT_SEMI, RIGHT_ANTI, or inner_unique join,
+	 * compute correction factors for cost estimation.  These will be the same
+	 * for all paths.
 	 */
-	if (jointype == JOIN_SEMI || jointype == JOIN_ANTI || extra.inner_unique)
+	if (jointype == JOIN_SEMI || jointype == JOIN_ANTI ||
+		jointype == JOIN_RIGHT_SEMI || jointype == JOIN_RIGHT_ANTI ||
+		extra.inner_unique)
 		compute_semi_anti_join_factors(root, joinrel, outerrel, innerrel,
 									   jointype, sjinfo, restrictlist,
 									   &extra.semifactors);
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index d9650315016..c48e656ce80 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -3581,20 +3581,23 @@ typedef struct PlannerParamItem
 } PlannerParamItem;
 
 /*
- * When making cost estimates for a SEMI/ANTI/inner_unique join, there are
- * some correction factors that are needed in both nestloop and hash joins
- * to account for the fact that the executor can stop scanning inner rows
- * as soon as it finds a match to the current outer row.  These numbers
- * depend only on the selected outer and inner join relations, not on the
- * particular paths used for them, so it's worthwhile to calculate them
- * just once per relation pair not once per considered path.  This struct
- * is filled by compute_semi_anti_join_factors and must be passed along
- * to the join cost estimation functions.
- *
- * outer_match_frac is the fraction of the outer tuples that are
- *		expected to have at least one match.
+ * When making cost estimates for a SEMI/ANTI/RIGHT_SEMI/RIGHT_ANTI/
+ * inner_unique join, there are some correction factors that are needed in
+ * both nestloop and hash joins to account for the fact that the executor
+ * can stop scanning inner rows as soon as it finds a match to the current
+ * outer row.  These numbers depend only on the selected outer and inner
+ * join relations, not on the particular paths used for them, so it's
+ * worthwhile to calculate them just once per relation pair not once per
+ * considered path.  This struct is filled by
+ * compute_semi_anti_join_factors and must be passed along to the join
+ * cost estimation functions.
+ *
+ * outer_match_frac is the fraction of the semijoin's LHS tuples (the
+ *		physically inner side for RIGHT_SEMI/RIGHT_ANTI, the outer side
+ *		otherwise) that are expected to have at least one match.
  * match_count is the average number of matches expected for
- *		outer tuples that have at least one match.
+ *		outer tuples that have at least one match (not meaningful for
+ *		RIGHT_SEMI/RIGHT_ANTI).
  */
 typedef struct SemiAntiJoinFactors
 {
@@ -3612,7 +3615,8 @@ typedef struct SemiAntiJoinFactors
  * inner_unique is true if each outer tuple provably matches no more
  *		than one inner tuple
  * sjinfo is extra info about special joins for selectivity estimation
- * semifactors is as shown above (only valid for SEMI/ANTI/inner_unique joins)
+ * semifactors is as shown above (only valid for SEMI/ANTI/RIGHT_SEMI/
+ *		RIGHT_ANTI/inner_unique joins)
  * param_source_rels are OK targets for parameterization of result paths
  * pgs_mask is a bitmask of PGS_* constants to limit the join strategy
  */
diff --git a/src/test/regress/expected/join.out b/src/test/regress/expected/join.out
index 8bc75d349a9..e6165c2775f 100644
--- a/src/test/regress/expected/join.out
+++ b/src/test/regress/expected/join.out
@@ -1909,19 +1909,19 @@ select * from tenk1 a, tenk1 b
 where exists(select * from tenk1 c
              where b.twothousand = c.twothousand and b.fivethous <> c.fivethous)
       and a.tenthous = b.tenthous and a.tenthous < 5000;
-                  QUERY PLAN                   
------------------------------------------------
- Hash Semi Join
-   Hash Cond: (b.twothousand = c.twothousand)
+                     QUERY PLAN                      
+-----------------------------------------------------
+ Hash Right Semi Join
+   Hash Cond: (c.twothousand = b.twothousand)
    Join Filter: (b.fivethous <> c.fivethous)
-   ->  Hash Join
-         Hash Cond: (b.tenthous = a.tenthous)
-         ->  Seq Scan on tenk1 b
-         ->  Hash
-               ->  Seq Scan on tenk1 a
-                     Filter: (tenthous < 5000)
+   ->  Seq Scan on tenk1 c
    ->  Hash
-         ->  Seq Scan on tenk1 c
+         ->  Hash Join
+               Hash Cond: (b.tenthous = a.tenthous)
+               ->  Seq Scan on tenk1 b
+               ->  Hash
+                     ->  Seq Scan on tenk1 a
+                           Filter: (tenthous < 5000)
 (11 rows)
 
 --
@@ -3106,6 +3106,60 @@ and t1.fivethous < 5;
                ->  Parallel Seq Scan on tenk1 t2
 (8 rows)
 
+rollback;
+--
+-- Check that for hash right semi and right anti joins we charge cpu_tuple_cost
+-- and qp_qual_cost on the rows from the inner side.
+--
+begin;
+create temp table hj_small(id int primary key);
+create temp table hj_large(v int);
+insert into hj_small select i from generate_series(1,200)i;
+insert into hj_large select (i % 500) + 11 from generate_series(1,1000)i;
+analyze hj_small, hj_large;
+-- ensure we hash the small side and scan the large one, not the reverse
+explain (costs off)
+select count(*) from hj_small s where exists
+  (select 1 from hj_large r where r.v = s.id);
+                QUERY PLAN                
+------------------------------------------
+ Aggregate
+   ->  Hash Right Semi Join
+         Hash Cond: (r.v = s.id)
+         ->  Seq Scan on hj_large r
+         ->  Hash
+               ->  Seq Scan on hj_small s
+(6 rows)
+
+-- and check we get the expected results
+select count(*) from hj_small s where exists
+  (select 1 from hj_large r where r.v = s.id);
+ count 
+-------
+   190
+(1 row)
+
+-- likewise for a right anti join
+explain (costs off)
+select count(*) from hj_small s where not exists
+  (select 1 from hj_large r where r.v = s.id);
+                QUERY PLAN                
+------------------------------------------
+ Aggregate
+   ->  Hash Right Anti Join
+         Hash Cond: (r.v = s.id)
+         ->  Seq Scan on hj_large r
+         ->  Hash
+               ->  Seq Scan on hj_small s
+(6 rows)
+
+select count(*) from hj_small s where not exists
+  (select 1 from hj_large r where r.v = s.id);
+ count 
+-------
+    10
+(1 row)
+
 rollback;
 --
 -- regression test for bug #13908 (hash join with skew tuples & nbatch increase)
diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out
index 6b519a65cc9..67cae397861 100644
--- a/src/test/regress/expected/opr_sanity.out
+++ b/src/test/regress/expected/opr_sanity.out
@@ -1035,16 +1035,16 @@ WHERE c.castmethod = 'b' AND
                     k.casttarget = c.castsource);
     castsource     |    casttarget     | castfunc | castcontext 
 -------------------+-------------------+----------+-------------
- text              | character         |        0 | i
+ xml               | character varying |        0 | a
+ xml               | text              |        0 | a
  character varying | character         |        0 | i
- pg_node_tree      | text              |        0 | i
- pg_ndistinct      | bytea             |        0 | i
- pg_dependencies   | bytea             |        0 | i
+ text              | character         |        0 | i
+ xml               | character         |        0 | a
  pg_mcv_list       | bytea             |        0 | i
  cidr              | inet              |        0 | i
- xml               | text              |        0 | a
- xml               | character varying |        0 | a
- xml               | character         |        0 | a
+ pg_dependencies   | bytea             |        0 | i
+ pg_node_tree      | text              |        0 | i
+ pg_ndistinct      | bytea             |        0 | i
 (10 rows)
 
 -- **************** pg_conversion ****************
diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out
index 933921d1860..e1344215644 100644
--- a/src/test/regress/expected/select_parallel.out
+++ b/src/test/regress/expected/select_parallel.out
@@ -1125,27 +1125,27 @@ reset role;
 explain (costs off, verbose)
   select count(*) from tenk1 a where (unique1, two) in
     (select unique1, row_number() over() from tenk1 b);
-                                       QUERY PLAN                                       
-----------------------------------------------------------------------------------------
+                                          QUERY PLAN                                          
+----------------------------------------------------------------------------------------------
  Aggregate
    Output: count(*)
-   ->  Hash Right Semi Join
-         Hash Cond: ((b.unique1 = a.unique1) AND ((row_number() OVER w1) = a.two))
-         ->  WindowAgg
-               Output: b.unique1, row_number() OVER w1
-               Window: w1 AS (ROWS UNBOUNDED PRECEDING)
-               ->  Gather
-                     Output: b.unique1
-                     Workers Planned: 4
-                     ->  Parallel Index Only Scan using tenk1_unique1 on public.tenk1 b
-                           Output: b.unique1
-         ->  Hash
+   ->  Hash Semi Join
+         Hash Cond: ((a.unique1 = b.unique1) AND (a.two = (row_number() OVER w1)))
+         ->  Gather
                Output: a.unique1, a.two
-               ->  Gather
+               Workers Planned: 4
+               ->  Parallel Seq Scan on public.tenk1 a
                      Output: a.unique1, a.two
-                     Workers Planned: 4
-                     ->  Parallel Seq Scan on public.tenk1 a
-                           Output: a.unique1, a.two
+         ->  Hash
+               Output: b.unique1, (row_number() OVER w1)
+               ->  WindowAgg
+                     Output: b.unique1, row_number() OVER w1
+                     Window: w1 AS (ROWS UNBOUNDED PRECEDING)
+                     ->  Gather
+                           Output: b.unique1
+                           Workers Planned: 4
+                           ->  Parallel Index Only Scan using tenk1_unique1 on public.tenk1 b
+                                 Output: b.unique1
 (19 rows)
 
 -- LIMIT/OFFSET within sub-selects can't be pushed to workers.
diff --git a/src/test/regress/sql/join.sql b/src/test/regress/sql/join.sql
index a80ce1c17a7..161048a7999 100644
--- a/src/test/regress/sql/join.sql
+++ b/src/test/regress/sql/join.sql
@@ -779,6 +779,38 @@ and t1.fivethous < 5;
 
 rollback;
 
+--
+-- Check that for hash right semi and right anti joins we charge cpu_tuple_cost
+-- and qp_qual_cost on the rows from the inner side.
+--
+
+begin;
+
+create temp table hj_small(id int primary key);
+create temp table hj_large(v int);
+insert into hj_small select i from generate_series(1,200)i;
+insert into hj_large select (i % 500) + 11 from generate_series(1,1000)i;
+analyze hj_small, hj_large;
+
+-- ensure we hash the small side and scan the large one, not the reverse
+explain (costs off)
+select count(*) from hj_small s where exists
+  (select 1 from hj_large r where r.v = s.id);
+
+-- and check we get the expected results
+select count(*) from hj_small s where exists
+  (select 1 from hj_large r where r.v = s.id);
+
+-- likewise for a right anti join
+explain (costs off)
+select count(*) from hj_small s where not exists
+  (select 1 from hj_large r where r.v = s.id);
+
+select count(*) from hj_small s where not exists
+  (select 1 from hj_large r where r.v = s.id);
+
+rollback;
+
 --
 -- regression test for bug #13908 (hash join with skew tuples & nbatch increase)
 --
-- 
2.37.1 (Apple Git-137.1)

