Assertion failure in REL9_5_STABLE

Hi All,

I stumbled upon this test case from the master that sets an assertion
failure (see stack trace at the end) on REL9_5_STABLE.

create temp table gstest4(id integer, v integer,
unhashable_col bit(4), unsortable_col xid);
insert into gstest4
values (1,1,b'0000','1'), (2,2,b'0001','1'),
(3,4,b'0010','2'), (4,8,b'0011','2'),
(5,16,b'0000','2'), (6,32,b'0001','2'),
(7,64,b'0010','1'), (8,128,b'0011','1');

-- mixed hashable/sortable cases
select unhashable_col, unsortable_col,
grouping(unhashable_col, unsortable_col),
count(*), sum(v)
from gstest4 group by grouping sets ((unhashable_col),(unsortable_col))
order by 3, 5;

I tested this with REL9_6_STABLE and it throws a more useful error message,
presumably because the code was refactored quite heavily for upper planner
changes.

ERROR: could not implement GROUP BY
DETAIL: Some of the datatypes only support hashing, while others only
support sorting.

I am attaching a patch that throws a similar ERROR during planning even for
9.5. AFAICS in presence of grouping sets, we always decide to use
sort-based implementation for grouping, but do not check if the columns
support ordering or not.

I did not test it for other older branches because grouping sets were
introduced in 9.5.

Thanks,
Pavan

(lldb) bt
* thread #1: tid = 0x0000, 0x00007fff9c1dfdda
libsystem_kernel.dylib`__pthread_kill + 10, stop reason = signal SIGSTOP
* frame #0: 0x00007fff9c1dfdda libsystem_kernel.dylib`__pthread_kill + 10
frame #1: 0x00007fff9c2cb787 libsystem_pthread.dylib`pthread_kill + 90
frame #2: 0x00007fff9c145420 libsystem_c.dylib`abort + 129
frame #3: 0x000000010f61a3f0
postgres`ExceptionalCondition(conditionName="!(sortOperators[i] != 0)",
errorType="BadArgument", fileName="tuplesort.c", lineNumber=657) + 128 at
assert.c:54
frame #4: 0x000000010f65a07d
postgres`tuplesort_begin_heap(tupDesc=0x00007fb1528194d8, nkeys=1,
attNums=0x00007fb15280e9e0, sortOperators=0x00007fb15280ea00,
sortCollations=0x00007fb15280ea20, nullsFirstFlags="", workMem=4096,
randomAccess='\0') + 509 at tuplesort.c:657
frame #5: 0x000000010f2e871d
postgres`initialize_phase(aggstate=0x00007fb152817828, newphase=0) + 477 at
nodeAgg.c:456
frame #6: 0x000000010f2e73f0
postgres`ExecInitAgg(node=0x00007fb1528062e8, estate=0x00007fb152817440,
eflags=16) + 2656 at nodeAgg.c:2223
frame #7: 0x000000010f2d18e1
postgres`ExecInitNode(node=0x00007fb1528062e8, estate=0x00007fb152817440,
eflags=16) + 865 at execProcnode.c:296
frame #8: 0x000000010f301231
postgres`ExecInitSort(node=0x00007fb152806400, estate=0x00007fb152817440,
eflags=16) + 225 at nodeSort.c:202
frame #9: 0x000000010f2d18a9
postgres`ExecInitNode(node=0x00007fb152806400, estate=0x00007fb152817440,
eflags=16) + 809 at execProcnode.c:286
frame #10: 0x000000010f2ccf20
postgres`InitPlan(queryDesc=0x00007fb152803c40, eflags=16) + 1520 at
execMain.c:957
frame #11: 0x000000010f2cc81f
postgres`standard_ExecutorStart(queryDesc=0x00007fb152803c40, eflags=16) +
591 at execMain.c:237
frame #12: 0x000000010f2cc5be
postgres`ExecutorStart(queryDesc=0x00007fb152803c40, eflags=0) + 62 at
execMain.c:139
frame #13: 0x000000010f48b112
postgres`PortalStart(portal=0x00007fb151836c40, params=0x0000000000000000,
eflags=0, snapshot=0x0000000000000000) + 722 at pquery.c:533
frame #14: 0x000000010f4871c1
postgres`exec_simple_query(query_string="select unhashable_col,
unsortable_col,\n grouping(unhashable_col, unsortable_col),\n
count(*), sum(v)\n from gstest4 group by grouping sets
((unhashable_col),(unsortable_col))\n order by 3, 5;") + 945 at
postgres.c:1065
frame #15: 0x000000010f486525 postgres`PostgresMain(argc=1,
argv=0x00007fb151801f00, dbname="postgres", username="pavan") + 2245 at
postgres.c:4051
frame #16: 0x000000010f3ef392
postgres`BackendRun(port=0x00007fb151700540) + 674 at postmaster.c:4254
frame #17: 0x000000010f3ee64d
postgres`BackendStartup(port=0x00007fb151700540) + 365 at postmaster.c:3928
frame #18: 0x000000010f3ed705 postgres`ServerLoop + 597 at
postmaster.c:1698
frame #19: 0x000000010f3eb066 postgres`PostmasterMain(argc=3,
argv=0x00007fb151403740) + 5414 at postmaster.c:1306
frame #20: 0x000000010f32bddf postgres`main(argc=3,
argv=0x00007fb151403740) + 751 at main.c:228
frame #21: 0x00007fff9c0b1255 libdyld.dylib`start + 1
frame #22: 0x00007fff9c0b1255 libdyld.dylib`start + 1

--
Pavan Deolasee http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Training & Services