From a662c85b18c126e67a07aa68c613a7555150ec02 Mon Sep 17 00:00:00 2001
From: Tomas Vondra <tomas@pgaddict.com>
Date: Thu, 7 Apr 2016 00:58:03 +0200
Subject: [PATCH 2/2] use fkeys in join estimation

---
 src/backend/optimizer/path/costsize.c | 360 +++++++++++++++++++++++++++++++++-
 src/include/optimizer/paths.h         |   2 +
 2 files changed, 352 insertions(+), 10 deletions(-)

diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 70a4c27..b403558 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -3888,6 +3888,347 @@ get_parameterized_joinrel_size(PlannerInfo *root, RelOptInfo *rel,
 }
 
 /*
+ * quals_match_foreign_key
+ *		Determines if the foreign key is matched by joinquals.
+ *
+ * Checks that there are conditions on all columns of the foreign key, matching
+ * the operator used by the foreign key etc. If such complete match is found,
+ * the function returns bitmap identifying the matching quals (0-based).
+ *
+ * Otherwise (no match at all or incomplete match), NULL is returned.
+ */
+static Bitmapset *
+quals_match_foreign_key(PlannerInfo *root, ForeignKeyOptInfo *fkinfo,
+						RelOptInfo *fkrel, RelOptInfo *foreignrel,
+						List *joinquals)
+{
+	int i;
+	int nkeys = fkinfo->nkeys;
+	Bitmapset *qualmatches = NULL;
+	Bitmapset *fkmatches = NULL;
+
+	/*
+	 * Loop over each column of the foreign key and build a bitmap index
+	 * of each joinqual which matches. Note that we don't stop when we find
+	 * the first match, as the expression could be duplicated in the
+	 * joinquals, and we want to generate a bitmap index which has bits set for
+	 * every matching join qual.
+	 */
+	for (i = 0; i < nkeys; i++)
+	{
+		ListCell *lc;
+		int quallstidx = -1;
+
+		foreach(lc, joinquals)
+		{
+			RestrictInfo   *rinfo;
+			OpExpr		   *clause;
+			Var			   *leftvar;
+			Var			   *rightvar;
+
+			quallstidx++;
+
+			/*
+			 * Technically we don't need to, but here we skip this qual if
+			 * we've matched it to part of the foreign key already. This
+			 * should prove to be a useful optimization when the quals appear
+			 * in the same order as the foreign key's keys. We need only bother
+			 * doing this when the foreign key is made up of more than 1 set
+			 * of columns, and we're not testing the first column.
+			 */
+			if (i > 0 && bms_is_member(quallstidx, qualmatches))
+				continue;
+
+			/*
+			 * Here since 'usefulquals' only contains bitmap indexes for quals
+			 * of type "var op var" we can safely skip checking this.
+			 */
+			rinfo = (RestrictInfo *) lfirst(lc);
+			clause = (OpExpr *) rinfo->clause;
+
+			/*
+			 * If the operator does not match then there's little point in
+			 * checking the operands
+			 */
+			if (clause->opno != fkinfo->conpfeqop[i])
+				continue;
+
+			leftvar = (Var *) get_leftop((Expr *) clause);
+			rightvar = (Var *) get_rightop((Expr *) clause);
+
+			/* Foreign keys only support Vars, so ignore anything more complex */
+			if (!IsA(leftvar, Var) || !IsA(rightvar, Var))
+				continue;
+
+			/*
+			 * For RestrictInfos built from an eclass we must consider each
+			 * member of the eclass as rinfo's operands may not belong to the
+			 * foreign key. For efficient tracking of which Vars we've found,
+			 * since we're only tracking 2 Vars, we use a bitmask. We can
+			 * safely finish searching when both of the least significant bits
+			 * are set.
+			 */
+			if (rinfo->parent_ec)
+			{
+				EquivalenceClass   *ec = rinfo->parent_ec;
+				ListCell		   *lc2;
+				int					foundvarmask = 0;
+
+				foreach(lc2, ec->ec_members)
+				{
+					EquivalenceMember *em = (EquivalenceMember *) lfirst(lc2);
+					Var *var = (Var *) em->em_expr;
+
+					if (!IsA(var, Var))
+						continue;
+
+					if (foreignrel->relid == var->varno &&
+						fkinfo->confkeys[i] == var->varattno)
+						foundvarmask |= 1;
+
+					else if (fkrel->relid == var->varno &&
+						fkinfo->conkeys[i] == var->varattno)
+						foundvarmask |= 2;
+
+					/*
+					 * Check if we've found both matches. If found we add
+					 * this qual to the matched list and mark this key as
+					 * matched too.
+					 */
+					if (foundvarmask == 3)
+					{
+						qualmatches = bms_add_member(qualmatches, quallstidx);
+						fkmatches = bms_add_member(fkmatches, i);
+						break;
+					}
+				}
+			}
+			else
+			{
+				/*
+				 * In this non eclass RestrictInfo case we'll check if the left
+				 * and right Vars match to this part of the foreign key.
+				 * Remember that this could be written with the Vars in either
+				 * order, so we test both permutations of the expression.
+				 */
+				if ((foreignrel->relid == leftvar->varno) &&
+					(fkrel->relid == rightvar->varno) &&
+					(fkinfo->confkeys[i] == leftvar->varattno) &&
+					(fkinfo->conkeys[i] == rightvar->varattno))
+				{
+					qualmatches = bms_add_member(qualmatches, quallstidx);
+					fkmatches = bms_add_member(fkmatches, i);
+				}
+				else if ((foreignrel->relid == rightvar->varno) &&
+						 (fkrel->relid == leftvar->varno) &&
+						 (fkinfo->confkeys[i] == rightvar->varattno) &&
+						 (fkinfo->conkeys[i] == leftvar->varattno))
+				{
+					qualmatches = bms_add_member(qualmatches, quallstidx);
+					fkmatches = bms_add_member(fkmatches, i);
+				}
+			}
+		}
+	}
+
+	/* can't find more matches than columns in the foreign key */
+	Assert(bms_num_members(fkmatches) <= nkeys);
+
+	/* Only return the matches if we the foreign key is matched fully. */
+	if (bms_num_members(fkmatches) == nkeys)
+	{
+		bms_free(fkmatches);
+		return qualmatches;
+	}
+
+	bms_free(fkmatches);
+	bms_free(qualmatches);
+
+	return NULL;
+}
+
+/*
+ * find_best_foreign_key_quals
+ * 		Finds the foreign key best matching the joinquals.
+ *
+ * Analyzes joinquals to determine if any quals match foreign keys defined the
+ * two relations (fkrel referencing foreignrel). When multiple foreign keys
+ * match, we do choose the one with the most keys as the best one. We might
+ * also choose the one matching the most quals, however we assume the quals
+ * may be duplicated.
+ *
+ * We also track which joinquals match the current foreign key, so that we can
+ * easily skip then when computing the selectivity.
+ *
+ * When no matching foreign key is found we return 0, otherwise we return the
+ * number of keys in the foreign key.
+ *
+ * Foreign keys matched only partially ( are currently ignored.
+ */
+static int
+find_best_foreign_key_quals(PlannerInfo *root, RelOptInfo *fkrel,
+							RelOptInfo *foreignrel, List *joinquals,
+							Bitmapset **joinqualsbitmap)
+{
+	Bitmapset	   *qualbestmatch;
+	ListCell	   *lc;
+	int				bestmatchnkeys;
+
+	/* fast path out when there's no foreign keys on fkrel */
+	if (fkrel->fkeylist == NIL)
+	{
+		*joinqualsbitmap = NULL;
+		return 0;
+	}
+
+	qualbestmatch = NULL;
+	bestmatchnkeys = 0;
+
+	/* now check the matches for each foreign key defined on the fkrel */
+	foreach(lc, fkrel->fkeylist)
+	{
+		ForeignKeyOptInfo *fkinfo = (ForeignKeyOptInfo *) lfirst(lc);
+		Bitmapset *qualsmatched;
+
+		/*
+		 * We make no attempt in checking that this foreign key actually
+		 * references 'foreignrel', the reasoning here is that we may be able
+		 * to match the foreign key to an eclass member Var of a RestrictInfo
+		 * that's in qualslist, this Var may belong to some other relation.
+		 */
+		qualsmatched = quals_match_foreign_key(root, fkinfo, fkrel, foreignrel,
+											   joinquals);
+
+		/* Did we get a match? And is that match better than a previous one? */
+		if (qualsmatched != NULL && fkinfo->nkeys > bestmatchnkeys)
+		{
+			/* save the new best match */
+			bms_free(qualbestmatch);
+			qualbestmatch = qualsmatched;
+			bestmatchnkeys = fkinfo->nkeys;
+		}
+	}
+
+	*joinqualsbitmap = qualbestmatch;
+	return bestmatchnkeys;
+}
+
+/*
+ * clauselist_join_selectivity
+ *		Estimate selectivity of join clauses either by using foreign key info
+ *		or by using the regular clauselist_selectivity().
+ *
+ * Since selectivity estimates for each joinqual are multiplied together, this
+ * can cause significant underestimates on the number of join tuples in cases
+ * where there's more than 1 clause in the join condition. To help ease the
+ * pain here we make use of foreign keys, and we assume that 1 row will match
+ * when *all* of the foreign key columns are present in the join condition, any
+ * additional clauses are estimated using clauselist_selectivity().
+ */
+static Selectivity
+clauselist_join_selectivity(PlannerInfo *root, List *joinquals,
+							JoinType jointype, SpecialJoinInfo *sjinfo)
+{
+	int				outerid;
+	int				innerid;
+	Selectivity		sel = 1.0;
+	Bitmapset	   *foundfkquals = NULL;
+
+	innerid = -1;
+	while ((innerid = bms_next_member(sjinfo->min_righthand, innerid)) >= 0)
+	{
+		RelOptInfo *innerrel = find_base_rel(root, innerid);
+
+		outerid = -1;
+		while ((outerid = bms_next_member(sjinfo->min_lefthand, outerid)) >= 0)
+		{
+			RelOptInfo	   *outerrel = find_base_rel(root, outerid);
+			Bitmapset	   *outer2inner;
+			Bitmapset	   *inner2outer;
+			int				innermatches;
+			int				outermatches;
+
+			/*
+			 * check which quals are matched by a foreign key referencing the
+			 * innerrel.
+			 */
+			outermatches = find_best_foreign_key_quals(root, outerrel,
+											innerrel, joinquals, &outer2inner);
+
+			/* do the same, but with relations swapped */
+			innermatches = find_best_foreign_key_quals(root, innerrel,
+											outerrel, joinquals, &inner2outer);
+
+			/*
+			 * did we find any matches at all? If so we need to see which one is
+			 * the best/longest match
+			 */
+			if (outermatches != 0 || innermatches != 0)
+			{
+				double	referenced_tuples;
+				bool overlap;
+
+				/* either could be zero, but not both. */
+				if (outermatches < innermatches)
+				{
+					overlap = bms_overlap(foundfkquals, inner2outer);
+
+					foundfkquals = bms_add_members(foundfkquals, inner2outer);
+					referenced_tuples = Max(outerrel->tuples, 1.0);
+				}
+				else
+				{
+					overlap = bms_overlap(foundfkquals, outer2inner);
+
+					foundfkquals = bms_add_members(foundfkquals, outer2inner);
+					referenced_tuples = Max(innerrel->tuples, 1.0);
+				}
+
+				/*
+				 * XXX should we ignore these overlapping matches?
+				 * Or perhaps take the Max() or Min()?
+				 */
+				if (overlap)
+				{
+					if (jointype == JOIN_SEMI || jointype == JOIN_ANTI)
+						sel = Min(sel,Min(1.0 / (outerrel->tuples / Max(innerrel->tuples, 1.0)), 1.0));
+					else
+						sel = Min(sel, 1.0 / referenced_tuples);
+				}
+				else
+				{
+					if (jointype == JOIN_SEMI || jointype == JOIN_ANTI)
+						sel *= Min(1.0 / (outerrel->tuples / Max(innerrel->tuples, 1.0)), 1.0);
+					else
+						sel *= 1.0 / referenced_tuples;
+				}
+			}
+		}
+	}
+
+	/*
+	 * If any non matched quals exist then we build a list of the non-matches
+	 * and use clauselist_selectivity() to estimate the selectivity of these.
+	 */
+	if (bms_num_members(foundfkquals) < list_length(joinquals))
+	{
+		ListCell *lc;
+		int lstidx = 0;
+		List *nonfkeyclauses = NIL;
+
+		foreach (lc, joinquals)
+		{
+			if (!bms_is_member(lstidx, foundfkquals))
+				nonfkeyclauses = lappend(nonfkeyclauses, lfirst(lc));
+			lstidx++;
+		}
+		sel *= clauselist_selectivity(root, nonfkeyclauses, 0, jointype, sjinfo);
+	}
+
+	return sel;
+}
+
+/*
  * calc_joinrel_size_estimate
  *		Workhorse for set_joinrel_size_estimates and
  *		get_parameterized_joinrel_size.
@@ -3933,11 +4274,11 @@ calc_joinrel_size_estimate(PlannerInfo *root,
 		}
 
 		/* Get the separate selectivities */
-		jselec = clauselist_selectivity(root,
-										joinquals,
-										0,
-										jointype,
-										sjinfo);
+		jselec = clauselist_join_selectivity(root,
+											 joinquals,
+											 jointype,
+											 sjinfo);
+
 		pselec = clauselist_selectivity(root,
 										pushedquals,
 										0,
@@ -3950,11 +4291,10 @@ calc_joinrel_size_estimate(PlannerInfo *root,
 	}
 	else
 	{
-		jselec = clauselist_selectivity(root,
-										restrictlist,
-										0,
-										jointype,
-										sjinfo);
+		jselec = clauselist_join_selectivity(root,
+											 restrictlist,
+											 jointype,
+											 sjinfo);
 		pselec = 0.0;			/* not used, keep compiler quiet */
 	}
 
diff --git a/src/include/optimizer/paths.h b/src/include/optimizer/paths.h
index 7ad4f02..d9a3c1f 100644
--- a/src/include/optimizer/paths.h
+++ b/src/include/optimizer/paths.h
@@ -76,6 +76,8 @@ extern Expr *adjust_rowcompare_for_index(RowCompareExpr *clause,
 							int indexcol,
 							List **indexcolnos,
 							bool *var_on_left_p);
+extern bool has_matching_fkey(RelOptInfo *rel, RelOptInfo *frel, List *clauses,
+							  bool reverse);
 
 /*
  * tidpath.h
-- 
2.5.5

