diff --git a/src/backend/utils/adt/network_selfuncs.c b/src/backend/utils/adt/network_selfuncs.c index d0d806f..08ec945 100644 --- a/src/backend/utils/adt/network_selfuncs.c +++ b/src/backend/utils/adt/network_selfuncs.c @@ -1,32 +1,626 @@ /*------------------------------------------------------------------------- * * network_selfuncs.c * Functions for selectivity estimation of inet/cidr operators * - * Currently these are just stubs, but we hope to do better soon. + * Estimates are based on null fraction, distinct value count, most common + * values, and histogram of inet/cidr datatypes. * * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * * IDENTIFICATION * src/backend/utils/adt/network_selfuncs.c * *------------------------------------------------------------------------- */ #include "postgres.h" +#include + +#include "access/htup_details.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_statistic.h" +#include "utils/lsyscache.h" #include "utils/inet.h" +#include "utils/selfuncs.h" +/* Default selectivity constant for the inet overlap operator */ +#define DEFAULT_OVERLAP_SEL 0.01 + +/* Default selectivity constant for the other operators */ +#define DEFAULT_INCLUSION_SEL 0.005 + +/* Default selectivity for given operator */ +#define DEFAULT_SEL(operator) \ + ((operator) == OID_INET_OVERLAP_OP ? \ + DEFAULT_OVERLAP_SEL : DEFAULT_INCLUSION_SEL) + +static short int inet_opr_order(Oid operator, bool reversed); +static Selectivity inet_his_inclusion_selec(Datum *values, int nvalues, + Datum *constvalue, short int opr_order); +static Selectivity inet_mcv_join_selec(Datum *values1, float4 *numbers1, + int nvalues1, Datum *values2, float4 *numbers2, + int nvalues2, Oid operator, bool reversed); +static Selectivity inet_mcv_his_selec(Datum *mcv_values, float4 *mcv_numbers, + int mcv_nvalues, Datum *his_values, int his_nvalues, + int red_nvalues, short int opr_order, + Selectivity *max_selec_pointer); +static Selectivity inet_his_inclusion_join_selec(Datum *his1_values, + int his1_nvalues, Datum *his2_values, int his2_nvalues, + int red_nvalues, short int opr_order); +static short int inet_inclusion_cmp(inet *left, inet *right, + short int opr_order); +static short int inet_masklen_inclusion_cmp(inet *left, inet *right, + short int opr_order); +static short int inet_his_match_divider(inet *boundary, inet *query, + short int opr_order); + +/* + * Selectivity estimation for the subnet inclusion operators + */ Datum networksel(PG_FUNCTION_ARGS) { - PG_RETURN_FLOAT8(0.001); + PlannerInfo *root = (PlannerInfo *) PG_GETARG_POINTER(0); + Oid operator = PG_GETARG_OID(1); + List *args = (List *) PG_GETARG_POINTER(2); + int varRelid = PG_GETARG_INT32(3), + his_nvalues; + VariableStatData vardata; + Node *other; + bool varonleft; + Selectivity selec, + max_mcv_selec; + Datum constvalue, + *his_values; + Form_pg_statistic stats; + FmgrInfo proc; + + /* + * If expression is not (variable op something) or (something op + * variable), then punt and return a default estimate. + */ + if (!get_restriction_variable(root, args, varRelid, + &vardata, &other, &varonleft)) + PG_RETURN_FLOAT8(DEFAULT_SEL(operator)); + + /* + * Can't do anything useful if the something is not a constant, either. + */ + if (!IsA(other, Const)) + { + ReleaseVariableStats(vardata); + PG_RETURN_FLOAT8(DEFAULT_SEL(operator)); + } + + /* All of the subnet inclusion operators are strict. */ + if (((Const *) other)->constisnull) + { + ReleaseVariableStats(vardata); + PG_RETURN_FLOAT8(0.0); + } + + if (!HeapTupleIsValid(vardata.statsTuple)) + { + ReleaseVariableStats(vardata); + PG_RETURN_FLOAT8(DEFAULT_SEL(operator)); + } + + constvalue = ((Const *) other)->constvalue; + stats = (Form_pg_statistic) GETSTRUCT(vardata.statsTuple); + + fmgr_info(get_opcode(operator), &proc); + selec = mcv_selectivity(&vardata, &proc, constvalue, varonleft, + &max_mcv_selec); + + if (get_attstatsslot(vardata.statsTuple, + vardata.atttype, vardata.atttypmod, + STATISTIC_KIND_HISTOGRAM, InvalidOid, + NULL, + &his_values, &his_nvalues, + NULL, NULL)) + { + selec += (1.0 - stats->stanullfrac - max_mcv_selec) * + inet_his_inclusion_selec(his_values, his_nvalues, &constvalue, + inet_opr_order(operator, !varonleft)); + + free_attstatsslot(vardata.atttype, his_values, his_nvalues, NULL, 0); + } + else if (max_mcv_selec == 0) + selec = DEFAULT_SEL(operator); + + /* Result should be in range, but make sure... */ + CLAMP_PROBABILITY(selec); + + ReleaseVariableStats(vardata); + PG_RETURN_FLOAT8(selec); } +/* + * Join selectivity estimation for the subnet inclusion operators + * + * Calculates MCV vs MCV, MCV vs histogram and histogram vs histogram + * selectivity for join using the subnet inclusion operators. Unlike the + * join selectivity function for the equality operator, eqjoinsel(), 1 to 1 + * matching of the values is not enough. Network inclusion operators are + * likely to match many to many. It requires to loop the MVC and histogram + * lists to the end. Also, MCV vs histogram selectiviy is not neglected + * as in eqjoinsel(). + * + * To make the the function faster only some of the values from the first + * MVC and histogram matched to the second histogram. It is calculated by + * log2(). + */ Datum networkjoinsel(PG_FUNCTION_ARGS) { - PG_RETURN_FLOAT8(0.001); + PlannerInfo *root = (PlannerInfo *) PG_GETARG_POINTER(0); + Oid operator = PG_GETARG_OID(1); + List *args = (List *) PG_GETARG_POINTER(2); + SpecialJoinInfo *sjinfo = (SpecialJoinInfo *) PG_GETARG_POINTER(4); + VariableStatData vardata1, + vardata2; + Form_pg_statistic stats1, + stats2; + Selectivity selec, + mcv1_max_selec, + mcv2_max_selec; + bool reversed, + mcv1_exists, + mcv2_exists, + his1_exists, + his2_exists; + short int opr_order; + int mcv1_nvalues, + mcv2_nvalues, + mcv1_nnumbers, + mcv2_nnumbers, + his1_nvalues, + his2_nvalues, + red1_nvalues, + red2_nvalues; + Datum *mcv1_values, + *mcv2_values, + *his1_values, + *his2_values; + float4 *mcv1_numbers, + *mcv2_numbers; + + get_join_variables(root, args, sjinfo, &vardata1, &vardata2, &reversed); + + switch (sjinfo->jointype) + { + case JOIN_INNER: + case JOIN_LEFT: + case JOIN_FULL: + break; + default: + ReleaseVariableStats(vardata1); + ReleaseVariableStats(vardata2); + PG_RETURN_FLOAT8(DEFAULT_SEL(operator)); + } + + if (!HeapTupleIsValid(vardata1.statsTuple) || + !HeapTupleIsValid(vardata2.statsTuple)) + { + ReleaseVariableStats(vardata1); + ReleaseVariableStats(vardata2); + PG_RETURN_FLOAT8(DEFAULT_SEL(operator)); + } + + opr_order = inet_opr_order(operator, reversed); + stats1 = (Form_pg_statistic) GETSTRUCT(vardata1.statsTuple); + stats2 = (Form_pg_statistic) GETSTRUCT(vardata2.statsTuple); + mcv1_exists = get_attstatsslot(vardata1.statsTuple, + vardata1.atttype, vardata1.atttypmod, + STATISTIC_KIND_MCV, InvalidOid, + NULL, + &mcv1_values, &mcv1_nvalues, + &mcv1_numbers, &mcv1_nnumbers); + mcv2_exists = get_attstatsslot(vardata2.statsTuple, + vardata2.atttype, vardata2.atttypmod, + STATISTIC_KIND_MCV, InvalidOid, + NULL, + &mcv2_values, &mcv2_nvalues, + &mcv2_numbers, &mcv2_nnumbers); + his1_exists = get_attstatsslot(vardata1.statsTuple, + vardata1.atttype, vardata1.atttypmod, + STATISTIC_KIND_HISTOGRAM, InvalidOid, + NULL, + &his1_values, &his1_nvalues, + NULL, NULL); + his2_exists = get_attstatsslot(vardata2.statsTuple, + vardata2.atttype, vardata2.atttypmod, + STATISTIC_KIND_HISTOGRAM, InvalidOid, + NULL, + &his2_values, &his2_nvalues, + NULL, NULL); + + red1_nvalues = ((int) log2(Max(mcv1_nvalues, his1_nvalues))) + 1; + red2_nvalues = ((int) log2(Max(mcv2_nvalues, his2_nvalues))) + 1; + + selec = 0.0; + mcv1_max_selec = 0.0; + mcv2_max_selec = 0.0; + + if (mcv1_exists && mcv2_exists) + selec += inet_mcv_join_selec(mcv1_values, mcv1_numbers, mcv1_nvalues, + mcv2_values, mcv2_numbers, mcv2_nvalues, + operator, reversed); + if (mcv1_exists && his2_exists) + selec += inet_mcv_his_selec(mcv1_values, mcv1_numbers, mcv1_nvalues, + his2_values, his2_nvalues, + Min(mcv1_nvalues, red1_nvalues), + opr_order, &mcv1_max_selec); + if (mcv2_exists && his1_exists) + selec += inet_mcv_his_selec(mcv2_values, mcv2_numbers, mcv2_nvalues, + his1_values, his1_nvalues, + Min(mcv2_nvalues, red1_nvalues), + opr_order, &mcv2_max_selec); + if (his1_exists && his2_exists) + selec += (1.0 - stats1->stanullfrac - mcv1_max_selec) * + (1.0 - stats2->stanullfrac - mcv2_max_selec) * + inet_his_inclusion_join_selec(his1_values, his1_nvalues, + his2_values, his2_nvalues, + Min(his1_nvalues, red1_nvalues), + opr_order); + + if ((!mcv1_exists && !his1_exists) || (!mcv2_exists && !his2_exists)) + selec = DEFAULT_SEL(operator); + + if (mcv1_exists) + free_attstatsslot(vardata1.atttype, mcv1_values, mcv1_nvalues, + mcv1_numbers, mcv1_nnumbers); + if (mcv2_exists) + free_attstatsslot(vardata2.atttype, mcv2_values, mcv2_nvalues, + mcv2_numbers, mcv2_nnumbers); + if (his1_exists) + free_attstatsslot(vardata1.atttype, his1_values, his1_nvalues, NULL, 0); + if (his2_exists) + free_attstatsslot(vardata2.atttype, his2_values, his2_nvalues, NULL, 0); + ReleaseVariableStats(vardata1); + ReleaseVariableStats(vardata2); + + /* Result should be in range, but make sure... */ + CLAMP_PROBABILITY(selec); + + PG_RETURN_FLOAT8(selec); +} + +/* + * Practical comparable numbers for the subnet inclusion operators + */ +static short int +inet_opr_order(Oid operator, bool reversed) +{ + short int order; + + switch (operator) + { + case OID_INET_SUP_OP: + order = -2; + break; + case OID_INET_SUPEQ_OP: + order = -1; + break; + case OID_INET_OVERLAP_OP: + order = 0; + break; + case OID_INET_SUBEQ_OP: + order = 1; + break; + case OID_INET_SUB_OP: + order = 2; + break; + default: + elog(ERROR, "unknown operator for inet inclusion selectivity"); + } + + return (reversed ? order * -1 : order); +} + +/* + * Inet histogram inclusion selectivity estimation + * + * Calculates histogram selectivity for the subnet inclusion operators of + * the inet type. The return value is between 0 and 1. It should be + * corrected with the MVC selectivity and null fraction. If the constant + * is less than the first element or greater than the last element of + * the histogram the return value will be 0. + * + * The histogram is originally for the basic comparison operators. Only + * the common bits of the network part and the lenght of the network part + * (masklen) are appropriate for the subnet inclusion opeators. Fortunately, + * basic comparison fits in this situation. Even so, the lenght of the + * network part would not really be significant in the histogram. This would + * lead to big mistakes for data sets with uneven masklen distribution. + * To avoid this problem, comparison with the left and the right side of the + * buckets used together. + * + * Histogram bucket matches are calculated in two forms. If the constant + * matches both sides the bucket is considered as fully matched. If the + * constant matches only the right side the bucket, it is not considered + * as matched unless it is the last bucket, because it will match the next + * bucked. If all of these buckets would be considered as matched, it would + * lead to unfair multiple matches for some constants. + * + * The second form is to match the bucket partially. We try to calculate + * dividers for both of the boundaries. If the address family of the boundary + * does not match the constant or comparison of the lenght of the network + * parts is not true by the operator, the divider for the boundary would not + * taken into account. If both of the dividers can be calculated the greater + * one will be used to mimimize the mistake in the buckets which have + * disperate masklens. + * + * The divider on the partial bucket match is imagined as the distance + * between the decisive bits and the common bits of the addresses. It will + * be* used as power of two as it is the natural scale for the IP network + * inclusion. The partial bucket match divider calculation is an empirical + * formula and subject to change with more experiment. + * + * For partial match with buckets which have different address families + * on the left and right sides only the boundary with the same address + * family is taken into consideration. This can cause more mistakes for these + * buckets if the masklens of their boundaries are also disparate. It can + * only be the case for one bucket, if there are addresses with different + * families on the column. It seems as a better option than not considering + * these buckets. + */ +static Selectivity +inet_his_inclusion_selec(Datum *values, int nvalues, Datum *constvalue, + short int opr_order) +{ + inet *query, + *left, + *right; + float match; + int i; + short int left_order, + right_order, + left_divider, + right_divider; + + match = 0.0; + query = DatumGetInetP(*constvalue); + left = DatumGetInetP(values[0]); + left_order = inet_inclusion_cmp(left, query, opr_order); + + for (i = 1; i < nvalues; i++) + { + right = DatumGetInetP(values[i]); + right_order = inet_inclusion_cmp(right, query, opr_order); + + if (left_order == 0 && right_order == 0) + { + /* Full bucket match. */ + + match += 1.0; + } + else if ((left_order <= 0 && right_order > 0) || + (left_order >= 0 && right_order < 0) || + (right_order == 0 && i == nvalues - 1)) + { + /* Partial bucket match. */ + + left_divider = inet_his_match_divider(left, query, opr_order); + right_divider = inet_his_match_divider(right, query, opr_order); + + if (left_divider >= 0 || right_divider >= 0) + match += 1.0 / pow(2, Max(left_divider, right_divider)); + } + + /* Shift the variables. */ + left = right; + left_order = right_order; + } + + /* There are nvalues - 1 buckets. */ + return match / (nvalues - 1); +} + +/* + * Inet MCV join selectivity estimation + * + * The original function of the operator used in this function, like the + * mcv_selectivity() on selfuncs.c. Actually this function has nothing + * to do with the network data types except its name and location. + */ +static Selectivity +inet_mcv_join_selec(Datum *values1, float4 *numbers1, int nvalues1, + Datum *values2, float4 *numbers2, int nvalues2, + Oid operator, bool reversed) +{ + Selectivity selec; + FmgrInfo proc; + int i, + j; + + fmgr_info(get_opcode(operator), &proc); + selec = 0.0; + + for (i = 0; i < nvalues1; i++) + for (j = 0; j < nvalues2; j++) + if (reversed ? + DatumGetBool(FunctionCall2Coll(&proc, + DEFAULT_COLLATION_OID, + values1[i], + values2[j])) : + DatumGetBool(FunctionCall2Coll(&proc, + DEFAULT_COLLATION_OID, + values2[j], + values1[i]))) + selec += numbers1[i] * numbers2[j]; + + return selec; +} + +/* + * Inet MCV vs histogram inclusion join selectivity estimation + * + * The function result is the selectivity, and the fraction of the total + * population of the MCV is returned into *max_selec_pointer. + */ +static Selectivity +inet_mcv_his_selec(Datum *mcv_values, float4 *mcv_numbers, int mcv_nvalues, + Datum *his_values, int his_nvalues, int red_nvalues, + short int opr_order, Selectivity *max_selec_pointer) +{ + Selectivity selec, + red_selec, + max_selec; + int i; + + selec = 0.0; + red_selec = 0.0; + max_selec = 0.0; + + for (i = 0; i < mcv_nvalues; i++) + { + if (i < red_nvalues) + { + selec += mcv_numbers[i] * + inet_his_inclusion_selec(his_values, his_nvalues, + &mcv_values[i], opr_order); + + red_selec += mcv_numbers[i]; + } + + max_selec += mcv_numbers[i]; + } + + *max_selec_pointer = max_selec; + return selec * max_selec / red_selec; +} + +/* + * Inet histogram inclusion join selectivity estimation + * + * Selected values from the first histogram will be matched with the second. + * red_nvalues of the values will by discarding same amount of values from + * the begging and the end of the list, on the grounds that they are outliers + * and hence not very representative. + */ +static Selectivity +inet_his_inclusion_join_selec(Datum *his1_values, int his1_nvalues, + Datum *his2_values, int his2_nvalues, + int red_nvalues, short int opr_order) +{ + float match; + int nskip, + i; + + match = 0.0; + nskip = (his1_nvalues - red_nvalues) / 2; + + for (i = nskip; i < his1_nvalues - nskip; i++) + match += inet_his_inclusion_selec(his2_values, his2_nvalues, + &his1_values[i], opr_order); + + return match / (his1_nvalues - 2 * nskip); +} + +/* + * Comparison function for the subnet inclusion operators + * + * Comparison is compatible with the basic comparison function for the inet + * type. See network_cmp_internal on network.c for the original. Basic + * comparison operators are implemented with the network_cmp_internal + * function. It is possible to implement the subnet inclusion operators with + * this function. + * + * Comparison is first on the common bits of the network part, then on + * the length of the network part (masklen) as the network_cmp_internal + * function. Only the first part is on this function. The second part is + * seperated to another function for reusability. The difference between + * the second part and the original network_cmp_internal is that the operator + * is used while comparing the lengths of the network parts. See the second + * part on the inet_masklen_inclusion_cmp function below. + */ +static short int +inet_inclusion_cmp(inet *left, inet *right, short int opr_order) +{ + if (ip_family(left) == ip_family(right)) + { + short int order; + + order = bitncmp(ip_addr(left), ip_addr(right), + Min(ip_bits(left), ip_bits(right))); + + if (order != 0) + return order; + + return inet_masklen_inclusion_cmp(left, right, opr_order); + } + + return ip_family(left) - ip_family(right); +} + +/* + * Masklen comparison function for the subnet inclusion operators + * + * Compares the lengths of network parts of the inputs using the operator. + * If the comparision is okay for the operator the return value will be 0. + * Otherwise the return value will be less than or greater than 0 with + * respect to the operator. + */ +static short int +inet_masklen_inclusion_cmp(inet *left, inet *right, short int opr_order) +{ + if (ip_family(left) == ip_family(right)) + { + short int order; + + order = ip_bits(left) - ip_bits(right); + + if ((order > 0 && opr_order >= 0) || + (order == 0 && opr_order >= -1 && opr_order <= 1) || + (order < 0 && opr_order <= 0)) + return 0; + + return opr_order; + } + + return ip_family(left) - ip_family(right); +} + +/* + * Inet histogram partial match divider calculation + * + * First the families and the lenghts of the network parts are compared + * using the subnet inclusion operator. The divider will be calculated + * using the masklens and the common bits of the addresses. -1 will be + * returned if it cannot be calculated. + */ +static short int +inet_his_match_divider(inet *boundary, inet *query, short int opr_order) +{ + if (inet_masklen_inclusion_cmp(boundary, query, opr_order) == 0) + { + short int min_bits, + decisive_bits; + + min_bits = Min(ip_bits(boundary), ip_bits(query)); + + /* + * Set the decisive bits from the one which should contain the other + * according to the operator. + */ + if (opr_order < 0) + decisive_bits = ip_bits(boundary); + else if (opr_order > 0) + decisive_bits = ip_bits(query); + else + decisive_bits = min_bits; + + if (min_bits > 0) + return decisive_bits - bitncommon(ip_addr(boundary), ip_addr(query), + min_bits); + return decisive_bits; + } + + return -1; } diff --git a/src/include/catalog/pg_operator.h b/src/include/catalog/pg_operator.h index f8b4a65..fb37337 100644 --- a/src/include/catalog/pg_operator.h +++ b/src/include/catalog/pg_operator.h @@ -1135,32 +1135,33 @@ DESCR("not equal"); DATA(insert OID = 1203 ( "<" PGNSP PGUID b f f 869 869 16 1205 1206 network_lt scalarltsel scalarltjoinsel )); DESCR("less than"); DATA(insert OID = 1204 ( "<=" PGNSP PGUID b f f 869 869 16 1206 1205 network_le scalarltsel scalarltjoinsel )); DESCR("less than or equal"); DATA(insert OID = 1205 ( ">" PGNSP PGUID b f f 869 869 16 1203 1204 network_gt scalargtsel scalargtjoinsel )); DESCR("greater than"); DATA(insert OID = 1206 ( ">=" PGNSP PGUID b f f 869 869 16 1204 1203 network_ge scalargtsel scalargtjoinsel )); DESCR("greater than or equal"); DATA(insert OID = 931 ( "<<" PGNSP PGUID b f f 869 869 16 933 0 network_sub networksel networkjoinsel )); DESCR("is subnet"); -#define OID_INET_SUB_OP 931 +#define OID_INET_SUB_OP 931 DATA(insert OID = 932 ( "<<=" PGNSP PGUID b f f 869 869 16 934 0 network_subeq networksel networkjoinsel )); DESCR("is subnet or equal"); -#define OID_INET_SUBEQ_OP 932 +#define OID_INET_SUBEQ_OP 932 DATA(insert OID = 933 ( ">>" PGNSP PGUID b f f 869 869 16 931 0 network_sup networksel networkjoinsel )); DESCR("is supernet"); -#define OID_INET_SUP_OP 933 +#define OID_INET_SUP_OP 933 DATA(insert OID = 934 ( ">>=" PGNSP PGUID b f f 869 869 16 932 0 network_supeq networksel networkjoinsel )); DESCR("is supernet or equal"); -#define OID_INET_SUPEQ_OP 934 +#define OID_INET_SUPEQ_OP 934 DATA(insert OID = 3552 ( "&&" PGNSP PGUID b f f 869 869 16 3552 0 network_overlap networksel networkjoinsel )); DESCR("overlaps (is subnet or supernet)"); +#define OID_INET_OVERLAP_OP 3552 DATA(insert OID = 2634 ( "~" PGNSP PGUID l f f 0 869 869 0 0 inetnot - - )); DESCR("bitwise not"); DATA(insert OID = 2635 ( "&" PGNSP PGUID b f f 869 869 869 0 0 inetand - - )); DESCR("bitwise and"); DATA(insert OID = 2636 ( "|" PGNSP PGUID b f f 869 869 869 0 0 inetor - - )); DESCR("bitwise or"); DATA(insert OID = 2637 ( "+" PGNSP PGUID b f f 869 20 869 2638 0 inetpl - - )); DESCR("add"); DATA(insert OID = 2638 ( "+" PGNSP PGUID b f f 20 869 869 2637 0 int8pl_inet - - ));