diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 449b54f..d82f739 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -3300,6 +3300,97 @@ ANALYZE measurement; + + Column Store + + + column store + + + columnar storage + + + vertical partitioning + + + + By default, PostgreSQL places all columns + of a table together within a single main data structure, referred + to internally as the heap. + + + + Optionally, PostgreSQL allows you to + specify that columns can be held in secondary data structures, + known as column stores. Each column store has a unique name and + contains data only for its master table. If a column definition + specifies a column store then values of that column for all rows + of a table are stored only in the column store - and not within + the main heap. This is completely different from TOAST, which is + designed to handle oversized attributes by breaking them up into + chunks and storing them in a single common subtable for all columns. + It is possible to store multiple columns within the same column store. + + + + With this feature PostgreSQL can be described + as a hybrid row/column store. PostgreSQL + implements column stores by providing a flexible column store API, + allowing different types of storage to be designed for different datatypes + or for different use cases. + Column stores are also sometimes referred to as columnar storage or + vertical paritioning. This section describes why and how to implement column + stores or vertical partitioning as part of your database design. + + + + Column stores provide the following advantages + + + + By grouping related columns together, data not required for the current + query or action will be completely avoided, significantly reducing query + time. + + + + + By grouping related columns together it is possible to take advantage of + high compression rates, reducing data storage requirements and reducing + query times for very large databases. + + + + + By separating data into multiple column stores, datatype-specific + storage optimizations will become possible, further extending + PostgreSQL's support for custom datatypes. + + + + + By separating data into multiple subtables, it will be possible to + store tables that are much larger than the maximum table size for a single + heap (32TB). + + + + + Reusing storage technology from other projects and/or using technology + with different licencing restrictions (GPL licence etc). + + + + + Provides an architecture that may eventually allow us to have tables + with more than 1600 columns. + + + + + + + Foreign Data diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index f0c94d5..ca28308 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -22,8 +22,9 @@ PostgreSQL documentation CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXISTS ] table_name ( [ - { column_name data_type [ COLLATE collation ] [ column_constraint [ ... ] ] + { column_name data_type [ COLLATE collation ] [ column_constraint [ ... ] ] [ column_storage ] | table_constraint + | table_level_column_storage | LIKE source_table [ like_option ... ] } [, ... ] ] ) @@ -55,6 +56,14 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI [ ON DELETE action ] [ ON UPDATE action ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] +and column_store is: + + COLUMN STORE column_store_name + USING column_store_access_method + [ WITH ( storage_parameter + [= value] [, ... ] ) ] + [ TABLESPACE tablespace_name ] + and table_constraint is: [ CONSTRAINT constraint_name ] @@ -66,6 +75,15 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE action ] [ ON UPDATE action ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] +and table_level_column_store is: + + COLUMN STORE column_store_name + USING column_store_access_method + (column_name [, ...] ) + [ WITH ( storage_parameter + [= value] [, ... ] ) ] + [ TABLESPACE tablespace_name ] + and like_option is: { INCLUDING | EXCLUDING } { DEFAULTS | CONSTRAINTS | INDEXES | STORAGE | COMMENTS | ALL } @@ -262,6 +280,28 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI + COLUMN STORE column store name + + + The COLUMN STORE clause assigns a column to a named + column store with a column store access method defined by the + compulsory USING clause. Optional parameters may be set + in the WITH clause. A tablespace can be defined for each + column store by using the TABLESPACE clause, otherwise + the column store will use the same tablespace as the main table. + + + Column stores may contain multiple columns. Multi-column column stores + must be defined as table-level clauses. + + + Columns may be assigned to only one column store, otherwise they will + be stored as part of the main table. + + + + + INHERITS ( parent_table [, ... ] ) @@ -307,7 +347,8 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI - Column STORAGE settings are also copied from parent tables. + Column STORAGE settings are also copied from parent tables, + but not column storage definitions. @@ -320,6 +361,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI The LIKE clause specifies a table from which the new table automatically copies all column names, their data types, and their not-null constraints. + Column storage parameters are not copied by the LIKE clause. Unlike INHERITS, the new table and original table diff --git a/known-bugs b/known-bugs new file mode 100644 index 0000000..07591b2 --- /dev/null +++ b/known-bugs @@ -0,0 +1,13 @@ +Known Bugs, Issues and Limitations + +* Heisenbug: shared memory error from >100 column stores + Simon Riggs can reproduce this, no one else can. + +* ERROR: cache lookup failed for column store 160595 + This error shows up inexplicably when a table with over 100 column stores is + dropped. (Alvaro's guess is that it's related to pg_depend handling. Gotta + rework that and remove OIDs from pg_cstore). + +* If table P has column stores, and C inherits from it, there is no way to + "remove" the stores from C. A separate grammar clause will be needed for + this. diff --git a/schemas/tpc-h/create.sql b/schemas/tpc-h/create.sql new file mode 100644 index 0000000..1063b21 --- /dev/null +++ b/schemas/tpc-h/create.sql @@ -0,0 +1,85 @@ +CREATE TABLE supplier ( + s_suppkey INTEGER NOT NULL, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL, + s_phone CHAR(15) NOT NULL, + s_acctbal NUMERIC NOT NULL, + s_comment VARCHAR(101) NOT NULL +); + +CREATE TABLE part ( + p_partkey INTEGER NOT NULL, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size INTEGER NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice NUMERIC NOT NULL, + p_comment VARCHAR(23) NOT NULL +); + +CREATE TABLE partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost NUMERIC NOT NULL, + ps_comment VARCHAR(199) NOT NULL +); + +CREATE TABLE customer ( + c_custkey INTEGER NOT NULL, + c_name VARCHAR(25) NOT NULL, + c_address VARCHAR(40) NOT NULL, + c_nationkey INTEGER NOT NULL, + c_phone CHAR(15) NOT NULL, + c_acctbal NUMERIC NOT NULL, + c_mktsegment CHAR(10) NOT NULL, + c_comment VARCHAR(117) NOT NULL +); + +CREATE TABLE orders ( + o_orderkey BIGINT NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice NUMERIC NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL +); + +CREATE TABLE lineitem ( + l_orderkey BIGINT NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity NUMERIC NOT NULL, + l_extendedprice NUMERIC NOT NULL, + l_discount NUMERIC NOT NULL, + l_tax NUMERIC NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL +); + +CREATE TABLE nation ( + n_nationkey INTEGER NOT NULL, + n_name CHAR(25) NOT NULL, + n_regionkey INTEGER NOT NULL, + n_comment VARCHAR(152) NOT NULL +); + +CREATE TABLE region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) NOT NULL +); + diff --git a/schemas/tpc-h/create_fixeddecimal.sql b/schemas/tpc-h/create_fixeddecimal.sql new file mode 100644 index 0000000..6fbe036 --- /dev/null +++ b/schemas/tpc-h/create_fixeddecimal.sql @@ -0,0 +1,85 @@ +CREATE TABLE supplier ( + s_suppkey INTEGER NOT NULL, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL, + s_phone CHAR(15) NOT NULL, + s_acctbal FIXEDDECIMAL NOT NULL, + s_comment VARCHAR(101) NOT NULL +); + +CREATE TABLE part ( + p_partkey INTEGER NOT NULL, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size FIXEDDECIMAL NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice FIXEDDECIMAL NOT NULL, + p_comment VARCHAR(23) NOT NULL +); + +CREATE TABLE partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty FIXEDDECIMAL NOT NULL, + ps_supplycost FIXEDDECIMAL NOT NULL, + ps_comment VARCHAR(199) NOT NULL +); + +CREATE TABLE customer ( + c_custkey INTEGER NOT NULL, + c_name VARCHAR(25) NOT NULL, + c_address VARCHAR(40) NOT NULL, + c_nationkey INTEGER NOT NULL, + c_phone CHAR(15) NOT NULL, + c_acctbal FIXEDDECIMAL NOT NULL, + c_mktsegment CHAR(10) NOT NULL, + c_comment VARCHAR(117) NOT NULL +); + +CREATE TABLE orders ( + o_orderkey BIGINT NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice FIXEDDECIMAL NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL +); + +CREATE TABLE lineitem ( + l_orderkey BIGINT NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity FIXEDDECIMAL NOT NULL, + l_extendedprice FIXEDDECIMAL NOT NULL, + l_discount FIXEDDECIMAL NOT NULL, + l_tax FIXEDDECIMAL NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL +); + +CREATE TABLE nation ( + n_nationkey INTEGER NOT NULL, + n_name CHAR(25) NOT NULL, + n_regionkey INTEGER NOT NULL, + n_comment VARCHAR(152) NOT NULL +); + +CREATE TABLE region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) NOT NULL +); + diff --git a/schemas/tpc-h/create_schema_fixeddecimal_phase1.sql b/schemas/tpc-h/create_schema_fixeddecimal_phase1.sql new file mode 100644 index 0000000..ae4d797 --- /dev/null +++ b/schemas/tpc-h/create_schema_fixeddecimal_phase1.sql @@ -0,0 +1,92 @@ +CREATE COLUMN STORE ACCESS METHOD vertical HANDLER vertical_cstore_handler; + +CREATE TABLE supplier ( + s_suppkey INTEGER NOT NULL, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL, + s_phone CHAR(15) NOT NULL, + s_acctbal FIXEDDECIMAL NOT NULL, + s_comment VARCHAR(101) NOT NULL +); + +CREATE TABLE part ( + p_partkey INTEGER NOT NULL, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size FIXEDDECIMAL NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice FIXEDDECIMAL NOT NULL, + p_comment VARCHAR(23) NOT NULL +); + +CREATE TABLE partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty FIXEDDECIMAL NOT NULL, + ps_supplycost FIXEDDECIMAL NOT NULL, + ps_comment VARCHAR(199) NOT NULL, + COLUMN STORE comment USING vertical (ps_comment) +); + +CREATE TABLE customer ( + c_custkey INTEGER NOT NULL, + c_name VARCHAR(25) NOT NULL, + c_address VARCHAR(40) NOT NULL, + c_nationkey INTEGER NOT NULL, + c_phone CHAR(15) NOT NULL, + c_acctbal FIXEDDECIMAL NOT NULL, + c_mktsegment CHAR(10) NOT NULL, + c_comment VARCHAR(117) NOT NULL +); + +CREATE TABLE orders ( + o_orderkey BIGINT NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice FIXEDDECIMAL NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL, + COLUMN STORE clerk USING vertical (o_clerk) +); + + +CREATE TABLE lineitem ( + l_orderkey BIGINT NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity FIXEDDECIMAL NOT NULL, + l_extendedprice FIXEDDECIMAL NOT NULL, + l_discount FIXEDDECIMAL NOT NULL, + l_tax FIXEDDECIMAL NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + COLUMN STORE comment USING vertical (l_comment) +); + + +CREATE TABLE nation ( + n_nationkey INTEGER NOT NULL, + n_name CHAR(25) NOT NULL, + n_regionkey INTEGER NOT NULL, + n_comment VARCHAR(152) NOT NULL +); + +CREATE TABLE region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) NOT NULL +); + diff --git a/schemas/tpc-h/create_schema_fixeddecimal_phase2.sql b/schemas/tpc-h/create_schema_fixeddecimal_phase2.sql new file mode 100644 index 0000000..6619251 --- /dev/null +++ b/schemas/tpc-h/create_schema_fixeddecimal_phase2.sql @@ -0,0 +1,97 @@ +CREATE COLUMN STORE ACCESS METHOD vertical HANDLER vertical_cstore_handler; + +CREATE TABLE supplier ( + s_suppkey INTEGER NOT NULL, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL, + s_phone CHAR(15) NOT NULL, + s_acctbal FIXEDDECIMAL NOT NULL, + s_comment VARCHAR(101) NOT NULL +); + +CREATE TABLE part ( + p_partkey INTEGER NOT NULL, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size FIXEDDECIMAL NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice FIXEDDECIMAL NOT NULL, + p_comment VARCHAR(23) NOT NULL, + COLUMN STORE container USING vertical (p_container) +); + +CREATE TABLE partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty FIXEDDECIMAL NOT NULL, + ps_supplycost FIXEDDECIMAL NOT NULL, + ps_comment VARCHAR(199) NOT NULL, + COLUMN STORE comment USING vertical (ps_comment) +); + +CREATE TABLE customer ( + c_custkey INTEGER NOT NULL, + c_name VARCHAR(25) NOT NULL, + c_address VARCHAR(40) NOT NULL, + c_nationkey INTEGER NOT NULL, + c_phone CHAR(15) NOT NULL, + c_acctbal FIXEDDECIMAL NOT NULL, + c_mktsegment CHAR(10) NOT NULL, + c_comment VARCHAR(117) NOT NULL, + COLUMN STORE mktsegment USING vertical (c_mktsegment) +); + + +CREATE TABLE orders ( + o_orderkey BIGINT NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice FIXEDDECIMAL NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL, + COLUMN STORE clerk USING vertical (o_clerk), + COLUMN STORE orderstatus USING vertical (o_orderstatus), + COLUMN STORE comment USING vertical (o_comment) +); + + +CREATE TABLE lineitem ( + l_orderkey BIGINT NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity FIXEDDECIMAL NOT NULL, + l_extendedprice FIXEDDECIMAL NOT NULL, + l_discount FIXEDDECIMAL NOT NULL, + l_tax FIXEDDECIMAL NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + COLUMN STORE comment USING vertical (l_comment) +); + + +CREATE TABLE nation ( + n_nationkey INTEGER NOT NULL, + n_name CHAR(25) NOT NULL, + n_regionkey INTEGER NOT NULL, + n_comment VARCHAR(152) NOT NULL +); + +CREATE TABLE region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) NOT NULL +); + diff --git a/schemas/tpc-h/create_schema_phase1.sql b/schemas/tpc-h/create_schema_phase1.sql new file mode 100644 index 0000000..978bcd3 --- /dev/null +++ b/schemas/tpc-h/create_schema_phase1.sql @@ -0,0 +1,92 @@ +CREATE COLUMN STORE ACCESS METHOD vertical HANDLER vertical_cstore_handler; + +CREATE TABLE supplier ( + s_suppkey INTEGER NOT NULL, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL, + s_phone CHAR(15) NOT NULL, + s_acctbal NUMERIC NOT NULL, + s_comment VARCHAR(101) NOT NULL +); + +CREATE TABLE part ( + p_partkey INTEGER NOT NULL, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size INTEGER NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice NUMERIC NOT NULL, + p_comment VARCHAR(23) NOT NULL +); + +CREATE TABLE partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost NUMERIC NOT NULL, + ps_comment VARCHAR(199) NOT NULL, + COLUMN STORE comment USING vertical (ps_comment) +); + +CREATE TABLE customer ( + c_custkey INTEGER NOT NULL, + c_name VARCHAR(25) NOT NULL, + c_address VARCHAR(40) NOT NULL, + c_nationkey INTEGER NOT NULL, + c_phone CHAR(15) NOT NULL, + c_acctbal NUMERIC NOT NULL, + c_mktsegment CHAR(10) NOT NULL, + c_comment VARCHAR(117) NOT NULL +); + +CREATE TABLE orders ( + o_orderkey BIGINT NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice NUMERIC NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL, + COLUMN STORE clerk USING vertical (o_clerk) +); + + +CREATE TABLE lineitem ( + l_orderkey BIGINT NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity NUMERIC NOT NULL, + l_extendedprice NUMERIC NOT NULL, + l_discount NUMERIC NOT NULL, + l_tax NUMERIC NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + COLUMN STORE comment USING vertical (l_comment) +); + + +CREATE TABLE nation ( + n_nationkey INTEGER NOT NULL, + n_name CHAR(25) NOT NULL, + n_regionkey INTEGER NOT NULL, + n_comment VARCHAR(152) NOT NULL +); + +CREATE TABLE region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) NOT NULL +); + diff --git a/schemas/tpc-h/create_schema_phase2.sql b/schemas/tpc-h/create_schema_phase2.sql new file mode 100644 index 0000000..f1ed171 --- /dev/null +++ b/schemas/tpc-h/create_schema_phase2.sql @@ -0,0 +1,97 @@ +CREATE COLUMN STORE ACCESS METHOD vertical HANDLER vertical_cstore_handler; + +CREATE TABLE supplier ( + s_suppkey INTEGER NOT NULL, + s_name CHAR(25) NOT NULL, + s_address VARCHAR(40) NOT NULL, + s_nationkey INTEGER NOT NULL, + s_phone CHAR(15) NOT NULL, + s_acctbal NUMERIC NOT NULL, + s_comment VARCHAR(101) NOT NULL +); + +CREATE TABLE part ( + p_partkey INTEGER NOT NULL, + p_name VARCHAR(55) NOT NULL, + p_mfgr CHAR(25) NOT NULL, + p_brand CHAR(10) NOT NULL, + p_type VARCHAR(25) NOT NULL, + p_size INTEGER NOT NULL, + p_container CHAR(10) NOT NULL, + p_retailprice NUMERIC NOT NULL, + p_comment VARCHAR(23) NOT NULL, + COLUMN STORE container USING vertical (p_container) +); + +CREATE TABLE partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost NUMERIC NOT NULL, + ps_comment VARCHAR(199) NOT NULL, + COLUMN STORE comment USING vertical (ps_comment) +); + +CREATE TABLE customer ( + c_custkey INTEGER NOT NULL, + c_name VARCHAR(25) NOT NULL, + c_address VARCHAR(40) NOT NULL, + c_nationkey INTEGER NOT NULL, + c_phone CHAR(15) NOT NULL, + c_acctbal NUMERIC NOT NULL, + c_mktsegment CHAR(10) NOT NULL, + c_comment VARCHAR(117) NOT NULL, + COLUMN STORE mktsegment USING vertical (c_mktsegment) +); + + +CREATE TABLE orders ( + o_orderkey BIGINT NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice NUMERIC NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL, + COLUMN STORE clerk USING vertical (o_clerk), + COLUMN STORE orderstatus USING vertical (o_orderstatus), + COLUMN STORE comment USING vertical (o_comment) +); + + +CREATE TABLE lineitem ( + l_orderkey BIGINT NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity NUMERIC NOT NULL, + l_extendedprice NUMERIC NOT NULL, + l_discount NUMERIC NOT NULL, + l_tax NUMERIC NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + COLUMN STORE comment USING vertical (l_comment) +); + + +CREATE TABLE nation ( + n_nationkey INTEGER NOT NULL, + n_name CHAR(25) NOT NULL, + n_regionkey INTEGER NOT NULL, + n_comment VARCHAR(152) NOT NULL +); + +CREATE TABLE region ( + r_regionkey INTEGER NOT NULL, + r_name CHAR(25) NOT NULL, + r_comment VARCHAR(152) NOT NULL +); + diff --git a/schemas/tpc-h/indexes.sql b/schemas/tpc-h/indexes.sql new file mode 100644 index 0000000..891000b --- /dev/null +++ b/schemas/tpc-h/indexes.sql @@ -0,0 +1,39 @@ +CREATE TABLESPACE fast_random_access LOCATION '/home//fast_random_access' WITH (random_page_cost = 3); + +CREATE INDEX customer_c_mktsegment_c_custkey_idx ON customer (c_mktsegment, c_custkey) WITH (fillfactor = 100); +CREATE INDEX customer_c_nationkey_c_custkey_idx ON customer (c_nationkey, c_custkey) WITH (fillfactor = 100); +CREATE INDEX customer_ios_test1 ON customer (substring(c_phone from 1 for 2), c_acctbal, c_custkey) WITH (fillfactor = 100); +CREATE UNIQUE INDEX pk_customer ON customer (c_custkey) WITH (fillfactor = 100); +CREATE INDEX line_item_l_orderkey_l_suppkey_idx ON lineitem (l_orderkey, l_suppkey) WITH (fillfactor = 100) TABLESPACE fast_random_access; +CREATE INDEX lineitem_l_orderkey_idx_l_returnflag ON lineitem (l_orderkey) WITH (fillfactor = 100) WHERE l_returnflag = 'R'; +CREATE INDEX lineitem_l_orderkey_idx_part1 ON lineitem (l_orderkey, l_suppkey) WITH (fillfactor = 100) WHERE l_commitdate < l_receiptdate; +CREATE INDEX lineitem_l_orderkey_idx_part2 ON lineitem (l_orderkey) WITH (fillfactor = 100) WHERE l_commitdate < l_receiptdate AND l_shipdate < l_commitdate; +CREATE INDEX lineitem_l_partkey_l_quantity_l_shipmode_idx ON lineitem (l_partkey, l_quantity, l_shipmode) WITH (fillfactor = 100); +CREATE INDEX lineitem_l_partkey_l_suppkey_l_shipdate_l_quantity_idx ON lineitem (l_partkey, l_suppkey, l_shipdate, l_quantity) WITH (fillfactor = 100); +CREATE INDEX lineitem_l_shipdate_idx ON lineitem USING BRIN (l_shipdate); +CREATE UNIQUE INDEX pk_lineitem ON lineitem (l_orderkey, l_linenumber) WITH (fillfactor = 100); +CREATE UNIQUE INDEX pk_nation ON nation (n_nationkey) WITH (fillfactor = 100); +CREATE INDEX orders_o_custkey_idx ON orders (o_custkey) WITH (fillfactor = 100); +CREATE INDEX orders_o_orderkey_o_orderdate_idx ON orders (o_orderkey, o_orderdate) WITH (fillfactor = 100); +CREATE INDEX orders_o_orderdate_idx ON orders USING BRIN (o_orderdate); +CREATE UNIQUE INDEX pk_orders ON orders (o_orderkey) WITH (fillfactor = 100); +CREATE INDEX part_ios_test1 ON part USING btree (p_size, p_partkey, p_brand, p_type) WITH (fillfactor = 100); +CREATE INDEX part_p_container_p_brand_p_partkey_idx ON part(p_container, p_brand, p_partkey) WITH (fillfactor = 100); +CREATE INDEX part_p_size_idx ON part USING BRIN (p_size); +CREATE INDEX part_p_type_p_partkey_idx ON part(p_type, p_partkey) WITH (fillfactor = 100); +CREATE UNIQUE INDEX pk_part ON part (p_partkey) WITH (fillfactor = 100); +CREATE INDEX partsupp_ps_suppkey_idx ON partsupp (ps_suppkey) WITH (fillfactor = 100); +CREATE UNIQUE INDEX pk_partsupp ON partsupp (ps_partkey, ps_suppkey) WITH (fillfactor = 100); +CREATE UNIQUE INDEX pk_region ON region (r_regionkey) WITH (fillfactor = 100); +CREATE UNIQUE INDEX pk_supplier ON supplier (s_suppkey) WITH (fillfactor = 100); +CREATE INDEX supplier_s_nationkey_s_suppkey_idx ON supplier (s_nationkey, s_suppkey) WITH (fillfactor = 100); +CREATE INDEX supplier_s_suppkey_idx_like ON supplier (s_suppkey) WITH (fillfactor = 100) WHERE s_comment LIKE '%Customer%Complaints%'; + +ALTER TABLE customer ADD CONSTRAINT pk_customer PRIMARY KEY USING INDEX pk_customer; +ALTER TABLE lineitem ADD CONSTRAINT pk_lineitem PRIMARY KEY USING INDEX pk_lineitem; +ALTER TABLE nation ADD CONSTRAINT pk_nation PRIMARY KEY USING INDEX pk_nation; +ALTER TABLE orders ADD CONSTRAINT pk_orders PRIMARY KEY USING INDEX pk_orders; +ALTER TABLE part ADD CONSTRAINT pk_part PRIMARY KEY USING INDEX pk_part; +ALTER TABLE partsupp ADD CONSTRAINT pk_partsupp PRIMARY KEY USING INDEX pk_partsupp; +ALTER TABLE region ADD CONSTRAINT pk_region PRIMARY KEY USING INDEX pk_region; +ALTER TABLE supplier ADD CONSTRAINT pk_supplier PRIMARY KEY USING INDEX pk_supplier; diff --git a/schemas/tpc-h/queries/q01.sql b/schemas/tpc-h/queries/q01.sql new file mode 100644 index 0000000..9b617b4 --- /dev/null +++ b/schemas/tpc-h/queries/q01.sql @@ -0,0 +1,24 @@ +-- using 1433771997 as a seed to the RNG + + +select + l_returnflag, + l_linestatus, + sum(l_quantity) as sum_qty, + sum(l_extendedprice) as sum_base_price, + sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + avg(l_quantity) as avg_qty, + avg(l_extendedprice) as avg_price, + avg(l_discount) as avg_disc, + count(*) as count_order +from + lineitem +where + l_shipdate <= date '1998-12-01' - interval '71 days' +group by + l_returnflag, + l_linestatus +order by + l_returnflag, + l_linestatus; diff --git a/schemas/tpc-h/queries/q02.sql b/schemas/tpc-h/queries/q02.sql new file mode 100644 index 0000000..8a016d4 --- /dev/null +++ b/schemas/tpc-h/queries/q02.sql @@ -0,0 +1,47 @@ +-- using 1433771997 as a seed to the RNG + + +select + s_acctbal, + s_name, + n_name, + p_partkey, + p_mfgr, + s_address, + s_phone, + s_comment +from + part, + supplier, + partsupp, + nation, + region +where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and p_size = 38 + and p_type like '%TIN' + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'MIDDLE EAST' + and ps_supplycost = ( + select + min(ps_supplycost) + from + partsupp, + supplier, + nation, + region + where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'MIDDLE EAST' + ) +order by + s_acctbal desc, + n_name, + s_name, + p_partkey +LIMIT 100; diff --git a/schemas/tpc-h/queries/q03.sql b/schemas/tpc-h/queries/q03.sql new file mode 100644 index 0000000..4177753 --- /dev/null +++ b/schemas/tpc-h/queries/q03.sql @@ -0,0 +1,26 @@ +-- using 1433771997 as a seed to the RNG + + +select + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) as revenue, + o_orderdate, + o_shippriority +from + customer, + orders, + lineitem +where + c_mktsegment = 'FURNITURE' + and c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate < date '1995-03-29' + and l_shipdate > date '1995-03-29' +group by + l_orderkey, + o_orderdate, + o_shippriority +order by + revenue desc, + o_orderdate +LIMIT 10; diff --git a/schemas/tpc-h/queries/q04.sql b/schemas/tpc-h/queries/q04.sql new file mode 100644 index 0000000..7e1e2de --- /dev/null +++ b/schemas/tpc-h/queries/q04.sql @@ -0,0 +1,25 @@ +-- using 1433771997 as a seed to the RNG + + +select + o_orderpriority, + count(*) as order_count +from + orders +where + o_orderdate >= date '1997-07-01' + and o_orderdate < date '1997-07-01' + interval '3' month + and exists ( + select + * + from + lineitem + where + l_orderkey = o_orderkey + and l_commitdate < l_receiptdate + ) +group by + o_orderpriority +order by + o_orderpriority; + diff --git a/schemas/tpc-h/queries/q05.sql b/schemas/tpc-h/queries/q05.sql new file mode 100644 index 0000000..3c06403 --- /dev/null +++ b/schemas/tpc-h/queries/q05.sql @@ -0,0 +1,28 @@ +-- using 1433771997 as a seed to the RNG + + +select + n_name, + sum(l_extendedprice * (1 - l_discount)) as revenue +from + customer, + orders, + lineitem, + supplier, + nation, + region +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and l_suppkey = s_suppkey + and c_nationkey = s_nationkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'MIDDLE EAST' + and o_orderdate >= date '1994-01-01' + and o_orderdate < date '1994-01-01' + interval '1' year +group by + n_name +order by + revenue desc; + diff --git a/schemas/tpc-h/queries/q06.sql b/schemas/tpc-h/queries/q06.sql new file mode 100644 index 0000000..6f44015 --- /dev/null +++ b/schemas/tpc-h/queries/q06.sql @@ -0,0 +1,13 @@ +-- using 1433771997 as a seed to the RNG + + +select + sum(l_extendedprice * l_discount) as revenue +from + lineitem +where + l_shipdate >= date '1994-01-01' + and l_shipdate < date '1994-01-01' + interval '1' year + and l_discount between 0.08 - 0.01 and 0.08 + 0.01 + and l_quantity < 24; + diff --git a/schemas/tpc-h/queries/q07.sql b/schemas/tpc-h/queries/q07.sql new file mode 100644 index 0000000..b045830 --- /dev/null +++ b/schemas/tpc-h/queries/q07.sql @@ -0,0 +1,43 @@ +-- using 1433771997 as a seed to the RNG + + +select + supp_nation, + cust_nation, + l_year, + sum(volume) as revenue +from + ( + select + n1.n_name as supp_nation, + n2.n_name as cust_nation, + extract(year from l_shipdate) as l_year, + l_extendedprice * (1 - l_discount) as volume + from + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2 + where + s_suppkey = l_suppkey + and o_orderkey = l_orderkey + and c_custkey = o_custkey + and s_nationkey = n1.n_nationkey + and c_nationkey = n2.n_nationkey + and ( + (n1.n_name = 'ROMANIA' and n2.n_name = 'INDIA') + or (n1.n_name = 'INDIA' and n2.n_name = 'ROMANIA') + ) + and l_shipdate between date '1995-01-01' and date '1996-12-31' + ) as shipping +group by + supp_nation, + cust_nation, + l_year +order by + supp_nation, + cust_nation, + l_year; + diff --git a/schemas/tpc-h/queries/q08.sql b/schemas/tpc-h/queries/q08.sql new file mode 100644 index 0000000..14de065 --- /dev/null +++ b/schemas/tpc-h/queries/q08.sql @@ -0,0 +1,41 @@ +-- using 1433771997 as a seed to the RNG + + +select + o_year, + sum(case + when nation = 'INDIA' then volume + else 0 + end) / sum(volume) as mkt_share +from + ( + select + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) as volume, + n2.n_name as nation + from + part, + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2, + region + where + p_partkey = l_partkey + and s_suppkey = l_suppkey + and l_orderkey = o_orderkey + and o_custkey = c_custkey + and c_nationkey = n1.n_nationkey + and n1.n_regionkey = r_regionkey + and r_name = 'ASIA' + and s_nationkey = n2.n_nationkey + and o_orderdate between date '1995-01-01' and date '1996-12-31' + and p_type = 'PROMO BRUSHED COPPER' + ) as all_nations +group by + o_year +order by + o_year; + diff --git a/schemas/tpc-h/queries/q09.sql b/schemas/tpc-h/queries/q09.sql new file mode 100644 index 0000000..3636569 --- /dev/null +++ b/schemas/tpc-h/queries/q09.sql @@ -0,0 +1,36 @@ +-- using 1433771997 as a seed to the RNG + + +select + nation, + o_year, + sum(amount) as sum_profit +from + ( + select + n_name as nation, + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + from + part, + supplier, + lineitem, + partsupp, + orders, + nation + where + s_suppkey = l_suppkey + and ps_suppkey = l_suppkey + and ps_partkey = l_partkey + and p_partkey = l_partkey + and o_orderkey = l_orderkey + and s_nationkey = n_nationkey + and p_name like '%yellow%' + ) as profit +group by + nation, + o_year +order by + nation, + o_year desc; + diff --git a/schemas/tpc-h/queries/q10.sql b/schemas/tpc-h/queries/q10.sql new file mode 100644 index 0000000..8e1b376 --- /dev/null +++ b/schemas/tpc-h/queries/q10.sql @@ -0,0 +1,35 @@ +-- using 1433771997 as a seed to the RNG + + +select + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) as revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment +from + customer, + orders, + lineitem, + nation +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate >= date '1994-01-01' + and o_orderdate < date '1994-01-01' + interval '3' month + and l_returnflag = 'R' + and c_nationkey = n_nationkey +group by + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment +order by + revenue desc +LIMIT 20; diff --git a/schemas/tpc-h/queries/q11.sql b/schemas/tpc-h/queries/q11.sql new file mode 100644 index 0000000..f48f0a6 --- /dev/null +++ b/schemas/tpc-h/queries/q11.sql @@ -0,0 +1,31 @@ +-- using 1433771997 as a seed to the RNG + + +select + ps_partkey, + sum(ps_supplycost * ps_availqty) as value +from + partsupp, + supplier, + nation +where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'ARGENTINA' +group by + ps_partkey having + sum(ps_supplycost * ps_availqty) > ( + select + sum(ps_supplycost * ps_availqty) * 0.0001000000 + from + partsupp, + supplier, + nation + where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'ARGENTINA' + ) +order by + value desc; + diff --git a/schemas/tpc-h/queries/q12.sql b/schemas/tpc-h/queries/q12.sql new file mode 100644 index 0000000..a4ff161 --- /dev/null +++ b/schemas/tpc-h/queries/q12.sql @@ -0,0 +1,32 @@ +-- using 1433771997 as a seed to the RNG + + +select + l_shipmode, + sum(case + when o_orderpriority = '1-URGENT' + or o_orderpriority = '2-HIGH' + then 1 + else 0 + end) as high_line_count, + sum(case + when o_orderpriority <> '1-URGENT' + and o_orderpriority <> '2-HIGH' + then 1 + else 0 + end) as low_line_count +from + orders, + lineitem +where + o_orderkey = l_orderkey + and l_shipmode in ('FOB', 'SHIP') + and l_commitdate < l_receiptdate + and l_shipdate < l_commitdate + and l_receiptdate >= date '1994-01-01' + and l_receiptdate < date '1994-01-01' + interval '1' year +group by + l_shipmode +order by + l_shipmode; + diff --git a/schemas/tpc-h/queries/q13.sql b/schemas/tpc-h/queries/q13.sql new file mode 100644 index 0000000..f0de62e --- /dev/null +++ b/schemas/tpc-h/queries/q13.sql @@ -0,0 +1,24 @@ +-- using 1433771997 as a seed to the RNG + + +select + c_count, + count(*) as custdist +from + ( + select + c_custkey, + count(o_orderkey) + from + customer left outer join orders on + c_custkey = o_custkey + and o_comment not like '%express%packages%' + group by + c_custkey + ) as c_orders (c_custkey, c_count) +group by + c_count +order by + custdist desc, + c_count desc; + diff --git a/schemas/tpc-h/queries/q14.sql b/schemas/tpc-h/queries/q14.sql new file mode 100644 index 0000000..d5681af --- /dev/null +++ b/schemas/tpc-h/queries/q14.sql @@ -0,0 +1,17 @@ +-- using 1433771997 as a seed to the RNG + + +select + 100.00 * sum(case + when p_type like 'PROMO%' + then l_extendedprice * (1 - l_discount) + else 0 + end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue +from + lineitem, + part +where + l_partkey = p_partkey + and l_shipdate >= date '1994-03-01' + and l_shipdate < date '1994-03-01' + interval '1' month; + diff --git a/schemas/tpc-h/queries/q15.sql b/schemas/tpc-h/queries/q15.sql new file mode 100644 index 0000000..44e9646 --- /dev/null +++ b/schemas/tpc-h/queries/q15.sql @@ -0,0 +1,37 @@ +-- using 1433771997 as a seed to the RNG + +create view revenue0 (supplier_no, total_revenue) as + select + l_suppkey, + sum(l_extendedprice * (1 - l_discount)) + from + lineitem + where + l_shipdate >= date '1993-01-01' + and l_shipdate < date '1993-01-01' + interval '3' month + group by + l_suppkey; + + +select + s_suppkey, + s_name, + s_address, + s_phone, + total_revenue +from + supplier, + revenue0 +where + s_suppkey = supplier_no + and total_revenue = ( + select + max(total_revenue) + from + revenue0 + ) +order by + s_suppkey; + +drop view revenue0; + diff --git a/schemas/tpc-h/queries/q16.sql b/schemas/tpc-h/queries/q16.sql new file mode 100644 index 0000000..727f13a --- /dev/null +++ b/schemas/tpc-h/queries/q16.sql @@ -0,0 +1,34 @@ +-- using 1433771997 as a seed to the RNG + + +select + p_brand, + p_type, + p_size, + count(distinct ps_suppkey) as supplier_cnt +from + partsupp, + part +where + p_partkey = ps_partkey + and p_brand <> 'Brand#45' + and p_type not like 'SMALL PLATED%' + and p_size in (19, 17, 16, 23, 10, 4, 38, 11) + and ps_suppkey not in ( + select + s_suppkey + from + supplier + where + s_comment like '%Customer%Complaints%' + ) +group by + p_brand, + p_type, + p_size +order by + supplier_cnt desc, + p_brand, + p_type, + p_size; + diff --git a/schemas/tpc-h/queries/q17.sql b/schemas/tpc-h/queries/q17.sql new file mode 100644 index 0000000..85ce2a7 --- /dev/null +++ b/schemas/tpc-h/queries/q17.sql @@ -0,0 +1,21 @@ +-- using 1433771997 as a seed to the RNG + + +select + sum(l_extendedprice) / 7.0 as avg_yearly +from + lineitem, + part +where + p_partkey = l_partkey + and p_brand = 'Brand#52' + and p_container = 'LG CAN' + and l_quantity < ( + select + 0.2 * avg(l_quantity) + from + lineitem + where + l_partkey = p_partkey + ); + diff --git a/schemas/tpc-h/queries/q18.sql b/schemas/tpc-h/queries/q18.sql new file mode 100644 index 0000000..77d692b --- /dev/null +++ b/schemas/tpc-h/queries/q18.sql @@ -0,0 +1,36 @@ +-- using 1433771997 as a seed to the RNG + + +select + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +from + customer, + orders, + lineitem +where + o_orderkey in ( + select + l_orderkey + from + lineitem + group by + l_orderkey having + sum(l_quantity) > 313 + ) + and c_custkey = o_custkey + and o_orderkey = l_orderkey +group by + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +order by + o_totalprice desc, + o_orderdate +LIMIT 100; diff --git a/schemas/tpc-h/queries/q19.sql b/schemas/tpc-h/queries/q19.sql new file mode 100644 index 0000000..61a6850 --- /dev/null +++ b/schemas/tpc-h/queries/q19.sql @@ -0,0 +1,39 @@ +-- using 1433771997 as a seed to the RNG + + +select + sum(l_extendedprice* (1 - l_discount)) as revenue +from + lineitem, + part +where + ( + p_partkey = l_partkey + and p_brand = 'Brand#22' + and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + and l_quantity >= 8 and l_quantity <= 8 + 10 + and p_size between 1 and 5 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#23' + and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + and l_quantity >= 10 and l_quantity <= 10 + 10 + and p_size between 1 and 10 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#12' + and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + and l_quantity >= 24 and l_quantity <= 24 + 10 + and p_size between 1 and 15 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ); + diff --git a/schemas/tpc-h/queries/q20.sql b/schemas/tpc-h/queries/q20.sql new file mode 100644 index 0000000..d092320 --- /dev/null +++ b/schemas/tpc-h/queries/q20.sql @@ -0,0 +1,41 @@ +-- using 1433771997 as a seed to the RNG + + +select + s_name, + s_address +from + supplier, + nation +where + s_suppkey in ( + select + ps_suppkey + from + partsupp + where + ps_partkey in ( + select + p_partkey + from + part + where + p_name like 'frosted%' + ) + and ps_availqty > ( + select + 0.5 * sum(l_quantity) + from + lineitem + where + l_partkey = ps_partkey + and l_suppkey = ps_suppkey + and l_shipdate >= date '1994-01-01' + and l_shipdate < date '1994-01-01' + interval '1' year + ) + ) + and s_nationkey = n_nationkey + and n_name = 'IRAN' +order by + s_name; + diff --git a/schemas/tpc-h/queries/q21.sql b/schemas/tpc-h/queries/q21.sql new file mode 100644 index 0000000..b796f2b --- /dev/null +++ b/schemas/tpc-h/queries/q21.sql @@ -0,0 +1,43 @@ +-- using 1433771997 as a seed to the RNG + + +select + s_name, + count(*) as numwait +from + supplier, + lineitem l1, + orders, + nation +where + s_suppkey = l1.l_suppkey + and o_orderkey = l1.l_orderkey + and o_orderstatus = 'F' + and l1.l_receiptdate > l1.l_commitdate + and exists ( + select + * + from + lineitem l2 + where + l2.l_orderkey = l1.l_orderkey + and l2.l_suppkey <> l1.l_suppkey + ) + and not exists ( + select + * + from + lineitem l3 + where + l3.l_orderkey = l1.l_orderkey + and l3.l_suppkey <> l1.l_suppkey + and l3.l_receiptdate > l3.l_commitdate + ) + and s_nationkey = n_nationkey + and n_name = 'GERMANY' +group by + s_name +order by + numwait desc, + s_name +LIMIT 100; diff --git a/schemas/tpc-h/queries/q22.sql b/schemas/tpc-h/queries/q22.sql new file mode 100644 index 0000000..c3c72a8 --- /dev/null +++ b/schemas/tpc-h/queries/q22.sql @@ -0,0 +1,41 @@ +-- using 1433771997 as a seed to the RNG + + +select + cntrycode, + count(*) as numcust, + sum(c_acctbal) as totacctbal +from + ( + select + substring(c_phone from 1 for 2) as cntrycode, + c_acctbal + from + customer + where + substring(c_phone from 1 for 2) in + ('30', '24', '31', '38', '25', '34', '37') + and c_acctbal > ( + select + avg(c_acctbal) + from + customer + where + c_acctbal > 0.00 + and substring(c_phone from 1 for 2) in + ('30', '24', '31', '38', '25', '34', '37') + ) + and not exists ( + select + * + from + orders + where + o_custkey = c_custkey + ) + ) as custsale +group by + cntrycode +order by + cntrycode; + diff --git a/src/backend/Makefile b/src/backend/Makefile index fb60420..4669ccb 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = access bootstrap catalog parser commands executor foreign lib libpq \ main nodes optimizer port postmaster regex replication rewrite \ - storage tcop tsearch utils $(top_builddir)/src/timezone + colstore storage tcop tsearch utils $(top_builddir)/src/timezone include $(srcdir)/common.mk diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index 94955b7..58666d1 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -79,7 +79,8 @@ /* * heap_compute_data_size - * Determine size of the data area of a tuple to be constructed + * Determine size of the data area of a tuple to be constructed with the + * attributes ordering in physical column ordering. */ Size heap_compute_data_size(TupleDesc tupleDesc, @@ -88,6 +89,64 @@ heap_compute_data_size(TupleDesc tupleDesc, { Size data_length = 0; int i; + int numberOfAttributes = tupleDesc->nphyatts; + Form_pg_attribute *att = tupleDesc->attrs; + + for (i = 0; i < numberOfAttributes; i++) + { + Datum val; + AttrNumber attnum = tupleDesc->attrmap[i]; + Form_pg_attribute atti; + + if (isnull[attnum - 1]) + continue; + + val = values[attnum - 1]; + atti = att[attnum - 1]; + + if (ATT_IS_PACKABLE(atti) && + VARATT_CAN_MAKE_SHORT(DatumGetPointer(val))) + { + /* + * we're anticipating converting to a short varlena header, so + * adjust length and don't count any alignment + */ + data_length += VARATT_CONVERTED_SHORT_SIZE(DatumGetPointer(val)); + } + else if (atti->attlen == -1 && + VARATT_IS_EXTERNAL_EXPANDED(DatumGetPointer(val))) + { + /* + * we want to flatten the expanded value so that the constructed + * tuple doesn't depend on it + */ + data_length = att_align_nominal(data_length, atti->attalign); + data_length += EOH_get_flat_size(DatumGetEOHP(val)); + } + else + { + data_length = att_align_datum(data_length, atti->attalign, + atti->attlen, val); + data_length = att_addlength_datum(data_length, atti->attlen, + val); + } + } + + return data_length; +} + +/* + * heap_compute_logical_data_size + * Determine size of the data area of a tuple to be constructed with the + * attributes layed out in logical column ordering. + */ +Size +heap_compute_logical_data_size(TupleDesc tupleDesc, + Datum *values, + bool *isnull) +{ + Size data_length = 0; + int i; int numberOfAttributes = tupleDesc->natts; Form_pg_attribute *att = tupleDesc->attrs; @@ -135,7 +194,8 @@ heap_compute_data_size(TupleDesc tupleDesc, /* * heap_fill_tuple - * Load data portion of a tuple from values/isnull arrays + * Load data portion of a tuple from values/isnull arrays in physical + * attribute order. * * We also fill the null bitmap (if any) and set the infomask bits * that reflect the tuple's data contents. @@ -151,6 +211,158 @@ heap_fill_tuple(TupleDesc tupleDesc, bits8 *bitP; int bitmask; int i; + int numberOfAttributes = tupleDesc->nphyatts; + Form_pg_attribute *att = tupleDesc->attrs; + +#ifdef USE_ASSERT_CHECKING + char *start = data; +#endif + + if (bit != NULL) + { + bitP = &bit[-1]; + bitmask = HIGHBIT; + } + else + { + /* just to keep compiler quiet */ + bitP = NULL; + bitmask = 0; + } + + *infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTERNAL); + + for (i = 0; i < numberOfAttributes; i++) + { + Size data_length; + AttrNumber attnum = tupleDesc->attrmap[i]; + Form_pg_attribute attr; + + attr = att[attnum - 1]; + + if (bit != NULL) + { + if (bitmask != HIGHBIT) + bitmask <<= 1; + else + { + bitP += 1; + *bitP = 0x0; + bitmask = 1; + } + + if (isnull[attnum - 1]) + { + *infomask |= HEAP_HASNULL; + continue; + } + + *bitP |= bitmask; + } + + /* + * XXX we use the att_align macros on the pointer value itself, not on + * an offset. This is a bit of a hack. + */ + + if (attr->attbyval) + { + /* pass-by-value */ + data = (char *) att_align_nominal(data, attr->attalign); + store_att_byval(data, values[attnum - 1], attr->attlen); + data_length = attr->attlen; + } + else if (attr->attlen == -1) + { + /* varlena */ + Pointer val = DatumGetPointer(values[attnum - 1]); + + *infomask |= HEAP_HASVARWIDTH; + if (VARATT_IS_EXTERNAL(val)) + { + if (VARATT_IS_EXTERNAL_EXPANDED(val)) + { + /* + * we want to flatten the expanded value so that the + * constructed tuple doesn't depend on it + */ + ExpandedObjectHeader *eoh = DatumGetEOHP(values[attnum - 1]); + + data = (char *) att_align_nominal(data, attr->attalign); + data_length = EOH_get_flat_size(eoh); + EOH_flatten_into(eoh, data, data_length); + } + else + { + *infomask |= HEAP_HASEXTERNAL; + /* no alignment, since it's short by definition */ + data_length = VARSIZE_EXTERNAL(val); + memcpy(data, val, data_length); + } + } + else if (VARATT_IS_SHORT(val)) + { + /* no alignment for short varlenas */ + data_length = VARSIZE_SHORT(val); + memcpy(data, val, data_length); + } + else if (VARLENA_ATT_IS_PACKABLE(attr) && + VARATT_CAN_MAKE_SHORT(val)) + { + /* convert to short varlena -- no alignment */ + data_length = VARATT_CONVERTED_SHORT_SIZE(val); + SET_VARSIZE_SHORT(data, data_length); + memcpy(data + 1, VARDATA(val), data_length - 1); + } + else + { + /* full 4-byte header varlena */ + data = (char *) att_align_nominal(data, attr->attalign); + data_length = VARSIZE(val); + memcpy(data, val, data_length); + } + } + else if (attr->attlen == -2) + { + /* cstring ... never needs alignment */ + *infomask |= HEAP_HASVARWIDTH; + Assert(attr->attalign == 'c'); + data_length = strlen(DatumGetCString(values[attnum - 1])) + 1; + memcpy(data, DatumGetPointer(values[attnum - 1]), data_length); + } + else + { + /* fixed-length pass-by-reference */ + data = (char *) att_align_nominal(data, attr->attalign); + Assert(attr->attlen > 0); + data_length = attr->attlen; + memcpy(data, DatumGetPointer(values[attnum - 1]), data_length); + } + + data += data_length; + } + + Assert((data - start) == data_size); +} + +/* + * heap_fill_logical_tuple + * Load data portion of a tuple from values/isnull arrays + * + * We also fill the null bitmap (if any) and set the infomask bits + * that reflect the tuple's data contents. + * + * NOTE: it is now REQUIRED that the caller have pre-zeroed the data area. + */ +static void +heap_fill_logical_tuple(TupleDesc tupleDesc, + Datum *values, bool *isnull, + char *data, Size data_size, + uint16 *infomask, bits8 *bit) +{ + bits8 *bitP; + int bitmask; + int i; int numberOfAttributes = tupleDesc->natts; Form_pg_attribute *att = tupleDesc->attrs; @@ -355,6 +567,227 @@ nocachegetattr(HeapTuple tuple, { HeapTupleHeader tup = tuple->t_data; Form_pg_attribute *att = tupleDesc->attrs; + AttrNumber phynum; + char *tp; /* ptr to data part of tuple */ + bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ + bool slow = false; /* do we have to walk attrs? */ + int off; /* current offset within data */ + + Assert(tupleDesc->attrs[attnum - 1]->attphynum != InvalidAttrNumber); + phynum = tupleDesc->attrs[attnum - 1]->attphynum - 1; + + /* ---------------- + * Three cases: + * + * 1: No nulls and no variable-width attributes. + * 2: Has a null or a var-width AFTER att. + * 3: Has nulls or var-widths BEFORE att. + * ---------------- + */ + + if (!HeapTupleNoNulls(tuple)) + { + /* + * there's a null somewhere in the tuple + * + * check to see if any preceding bits are null... + */ + int byte = phynum >> 3; + int finalbit = phynum & 0x07; + + /* check for nulls "before" final bit of last byte */ + if ((~bp[byte]) & ((1 << finalbit) - 1)) + slow = true; + else + { + /* check for nulls in any "earlier" bytes */ + int i; + + for (i = 0; i < byte; i++) + { + if (bp[i] != 0xFF) + { + slow = true; + break; + } + } + } + } + + tp = (char *) tup + tup->t_hoff; + + if (!slow) + { + /* + * If we get here, there are no nulls up to and including the target + * attribute. If we have a cached offset, we can use it. + */ + if (att[attnum - 1]->attcacheoff >= 0) + { + return fetchatt(att[attnum - 1], + tp + att[attnum - 1]->attcacheoff); + } + + /* + * Otherwise, check for non-fixed-length attrs up to and including + * target. If there aren't any, it's safe to cheaply initialize the + * cached offsets for these attrs. + */ + if (HeapTupleHasVarWidth(tuple)) + { + int j; + + for (j = 0; j <= phynum; j++) + { + Form_pg_attribute attr = att[tupleDesc->attrmap[j] - 1]; + if (attr->attlen <= 0) + { + slow = true; + break; + } + } + } + } + + if (!slow) + { + int natts = tupleDesc->nphyatts; + int j = 1; + + /* + * If we get here, we have a tuple with no nulls or var-widths up to + * and including the target attribute, so we can use the cached offset + * ... only we don't have it yet, or we'd not have got here. Since + * it's cheap to compute offsets for fixed-width columns, we take the + * opportunity to initialize the cached offsets for *all* the leading + * fixed-width columns, in hope of avoiding future visits to this + * routine. + */ + att[tupleDesc->attrmap[0] - 1]->attcacheoff = 0; + + /* we might have set some offsets in the slow path previously */ + while (j < natts && att[tupleDesc->attrmap[j] - 1]->attcacheoff > 0) + j++; + + off = att[tupleDesc->attrmap[j - 1] - 1]->attcacheoff + att[tupleDesc->attrmap[j - 1] - 1]->attlen; + + for (; j < natts; j++) + { + Form_pg_attribute attr = att[tupleDesc->attrmap[j] - 1]; + + if (attr->attlen <= 0) + break; + + off = att_align_nominal(off, attr->attalign); + + attr->attcacheoff = off; + + off += attr->attlen; + } + + Assert(j > phynum); + + off = att[attnum - 1]->attcacheoff; + } + else + { + bool usecache = true; + int i; + + /* + * Now we know that we have to walk the tuple CAREFULLY. But we still + * might be able to cache some offsets for next time. + * + * Note - This loop is a little tricky. For each non-null attribute, + * we have to first account for alignment padding before the attr, + * then advance over the attr based on its length. Nulls have no + * storage and no alignment padding either. We can use/set + * attcacheoff until we reach either a null or a var-width attribute. + */ + off = 0; + for (i = 0;; i++) /* loop exit is at "break" */ + { + int attno = tupleDesc->attrmap[i] - 1; + + if (HeapTupleHasNulls(tuple) && att_isnull(i, bp)) + { + usecache = false; + continue; /* this cannot be the target att */ + } + + /* If we know the next offset, we can skip the rest */ + if (usecache && att[attno]->attcacheoff >= 0) + off = att[attno]->attcacheoff; + else if (att[attno]->attlen == -1) + { + /* + * We can only cache the offset for a varlena attribute if the + * offset is already suitably aligned, so that there would be + * no pad bytes in any case: then the offset will be valid for + * either an aligned or unaligned value. + */ + if (usecache && + off == att_align_nominal(off, att[attno]->attalign)) + att[attno]->attcacheoff = off; + else + { + off = att_align_pointer(off, att[attno]->attalign, -1, + tp + off); + usecache = false; + } + } + else + { + /* not varlena, so safe to use att_align_nominal */ + off = att_align_nominal(off, att[attno]->attalign); + + if (usecache) + att[attno]->attcacheoff = off; + } + + if (i == phynum) + break; + + off = att_addlength_pointer(off, att[attno]->attlen, tp + off); + + if (usecache && att[attno]->attlen <= 0) + usecache = false; + } + } + + return fetchatt(att[attnum - 1], tp + off); +} + +/* ---------------- + * nocachegetlogattr + * + * This only gets called from fastgetlogattr() macro, in cases where + * we can't use a cacheoffset and the value is not null. + * + * This caches attribute offsets in the attribute descriptor. + * + * An alternative way to speed things up would be to cache offsets + * with the tuple, but that seems more difficult unless you take + * the storage hit of actually putting those offsets into the + * tuple you send to disk. Yuck. + * + * This scheme will be slightly slower than that, but should + * perform well for queries which hit large #'s of tuples. After + * you cache the offsets once, examining all the other tuples using + * the same attribute descriptor will go much quicker. -cim 5/4/91 + * + * NOTE: if you need to change this code, see also heap_deform_tuple. + * Also see nocache_index_getattr, which is the same code for index + * tuples. + * ---------------- + */ +Datum +nocachegetlogattr(HeapTuple tuple, + int attnum, + TupleDesc tupleDesc) +{ + HeapTupleHeader tup = tuple->t_data; + Form_pg_attribute *att = tupleDesc->attrs; char *tp; /* ptr to data part of tuple */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ bool slow = false; /* do we have to walk attrs? */ @@ -468,7 +901,7 @@ nocachegetattr(HeapTuple tuple, off += att[j]->attlen; } - Assert(j > attnum); + Assert(j >= attnum); off = att[attnum]->attcacheoff; } @@ -684,7 +1117,10 @@ heap_copy_tuple_as_datum(HeapTuple tuple, TupleDesc tupleDesc) /* * heap_form_tuple * construct a tuple from the given values[] and isnull[] arrays, - * which are of the length indicated by tupleDescriptor->natts + * which are of the length indicated by tupleDescriptor->natts. + * values[] and isnull[] are expected to be in the logical attribute + * (attnum) order, however the actual tuple is formed in physical + * attribute order. * * The result is allocated in the current memory context. */ @@ -699,7 +1135,7 @@ heap_form_tuple(TupleDesc tupleDescriptor, data_len; int hoff; bool hasnull = false; - int numberOfAttributes = tupleDescriptor->natts; + int numberOfAttributes = tupleDescriptor->nphyatts; int i; if (numberOfAttributes > MaxTupleAttributeNumber) @@ -713,7 +1149,8 @@ heap_form_tuple(TupleDesc tupleDescriptor, */ for (i = 0; i < numberOfAttributes; i++) { - if (isnull[i]) + AttrNumber phynum = tupleDescriptor->attrmap[i] - 1; + if (isnull[phynum]) { hasnull = true; break; @@ -777,6 +1214,101 @@ heap_form_tuple(TupleDesc tupleDescriptor, } /* + * heap_form_logical_tuple + * construct a tuple from the given values[] and isnull[] arrays, + * which are of the length indicated by tupleDescriptor->natts + * + * The result is allocated in the current memory context. + */ +HeapTuple +heap_form_logical_tuple(TupleDesc tupleDescriptor, + Datum *values, + bool *isnull) +{ + HeapTuple tuple; /* return tuple */ + HeapTupleHeader td; /* tuple data */ + Size len, + data_len; + int hoff; + bool hasnull = false; + int numberOfAttributes = tupleDescriptor->natts; + int i; + + if (numberOfAttributes > MaxTupleAttributeNumber) + ereport(ERROR, + (errcode(ERRCODE_TOO_MANY_COLUMNS), + errmsg("number of columns (%d) exceeds limit (%d)", + numberOfAttributes, MaxTupleAttributeNumber))); + + /* + * Check for nulls + */ + for (i = 0; i < numberOfAttributes; i++) + { + if (isnull[i]) + { + hasnull = true; + break; + } + } + + /* + * Determine total space needed + */ + len = offsetof(HeapTupleHeaderData, t_bits); + + if (hasnull) + len += BITMAPLEN(numberOfAttributes); + + if (tupleDescriptor->tdhasoid) + len += sizeof(Oid); + + hoff = len = MAXALIGN(len); /* align user data safely */ + + data_len = heap_compute_logical_data_size(tupleDescriptor, values, isnull); + + len += data_len; + + /* + * Allocate and zero the space needed. Note that the tuple body and + * HeapTupleData management structure are allocated in one chunk. + */ + tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len); + tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE); + + /* + * And fill in the information. Note we fill the Datum fields even though + * this tuple may never become a Datum. This lets HeapTupleHeaderGetDatum + * identify the tuple type if needed. + */ + tuple->t_len = len; + ItemPointerSetInvalid(&(tuple->t_self)); + tuple->t_tableOid = InvalidOid; + + HeapTupleHeaderSetDatumLength(td, len); + HeapTupleHeaderSetTypeId(td, tupleDescriptor->tdtypeid); + HeapTupleHeaderSetTypMod(td, tupleDescriptor->tdtypmod); + /* We also make sure that t_ctid is invalid unless explicitly set */ + ItemPointerSetInvalid(&(td->t_ctid)); + + HeapTupleHeaderSetNatts(td, numberOfAttributes); + td->t_hoff = hoff; + + if (tupleDescriptor->tdhasoid) /* else leave infomask = 0 */ + td->t_infomask = HEAP_HASOID; + + heap_fill_logical_tuple(tupleDescriptor, + values, + isnull, + (char *) td + hoff, + data_len, + &td->t_infomask, + (hasnull ? td->t_bits : NULL)); + + return tuple; +} + +/* * heap_modify_tuple * form a new tuple from an old tuple and a set of replacement values. * @@ -870,9 +1402,9 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, HeapTupleHeader tup = tuple->t_data; bool hasnulls = HeapTupleHasNulls(tuple); Form_pg_attribute *att = tupleDesc->attrs; - int tdesc_natts = tupleDesc->natts; + int tdesc_natts = tupleDesc->nphyatts; int natts; /* number of atts to extract */ - int attnum; + int phynum; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ @@ -891,11 +1423,12 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, off = 0; - for (attnum = 0; attnum < natts; attnum++) + for (phynum = 0; phynum < natts; phynum++) { + AttrNumber attnum = tupleDesc->attrmap[phynum] - 1; Form_pg_attribute thisatt = att[attnum]; - if (hasnulls && att_isnull(attnum, bp)) + if (hasnulls && att_isnull(phynum, bp)) { values[attnum] = (Datum) 0; isnull[attnum] = true; @@ -946,8 +1479,9 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc, * If tuple doesn't have all the atts indicated by tupleDesc, read the * rest as null */ - for (; attnum < tdesc_natts; attnum++) + for (; phynum < tdesc_natts; phynum++) { + AttrNumber attnum = tupleDesc->attrmap[phynum] - 1; values[attnum] = (Datum) 0; isnull[attnum] = true; } @@ -974,7 +1508,7 @@ slot_deform_tuple(TupleTableSlot *slot, int natts) HeapTupleHeader tup = tuple->t_data; bool hasnulls = HeapTupleHasNulls(tuple); Form_pg_attribute *att = tupleDesc->attrs; - int attnum; + int phynum; char *tp; /* ptr to tuple data */ long off; /* offset in tuple data */ bits8 *bp = tup->t_bits; /* ptr to null bitmap in tuple */ @@ -984,8 +1518,8 @@ slot_deform_tuple(TupleTableSlot *slot, int natts) * Check whether the first call for this tuple, and initialize or restore * loop state. */ - attnum = slot->tts_nvalid; - if (attnum == 0) + phynum = slot->tts_nvalid; + if (phynum == 0) { /* Start from the first attribute */ off = 0; @@ -1000,19 +1534,25 @@ slot_deform_tuple(TupleTableSlot *slot, int natts) tp = (char *) tup + tup->t_hoff; - for (; attnum < natts; attnum++) + for (; phynum < natts; phynum++) { - Form_pg_attribute thisatt = att[attnum]; + AttrNumber attnum = tupleDesc->attrmap[phynum]; + Form_pg_attribute thisatt; - if (hasnulls && att_isnull(attnum, bp)) + if (attnum == InvalidAttrNumber) + continue; + + thisatt = att[attnum - 1]; + + if (hasnulls && att_isnull(phynum, bp)) { - values[attnum] = (Datum) 0; - isnull[attnum] = true; + values[attnum - 1] = (Datum) 0; + isnull[attnum - 1] = true; slow = true; /* can't use attcacheoff anymore */ continue; } - isnull[attnum] = false; + isnull[attnum - 1] = false; if (!slow && thisatt->attcacheoff >= 0) off = thisatt->attcacheoff; @@ -1043,7 +1583,7 @@ slot_deform_tuple(TupleTableSlot *slot, int natts) thisatt->attcacheoff = off; } - values[attnum] = fetchatt(thisatt, tp + off); + values[attnum - 1] = fetchatt(thisatt, tp + off); off = att_addlength_pointer(off, thisatt->attlen, tp + off); @@ -1054,7 +1594,7 @@ slot_deform_tuple(TupleTableSlot *slot, int natts) /* * Save state for next execution */ - slot->tts_nvalid = attnum; + slot->tts_nvalid = phynum; slot->tts_off = off; slot->tts_slow = slow; } @@ -1077,7 +1617,7 @@ slot_getattr(TupleTableSlot *slot, int attnum, bool *isnull) HeapTuple tuple = slot->tts_tuple; TupleDesc tupleDesc = slot->tts_tupleDescriptor; HeapTupleHeader tup; - + int phynum; /* * system attributes are handled by heap_getsysattr */ @@ -1090,10 +1630,12 @@ slot_getattr(TupleTableSlot *slot, int attnum, bool *isnull) return heap_getsysattr(tuple, attnum, tupleDesc, isnull); } + phynum = tupleDesc->attrs[attnum - 1]->attphynum; + /* * fast path if desired attribute already cached */ - if (attnum <= slot->tts_nvalid) + if (phynum <= slot->tts_nvalid) { *isnull = slot->tts_isnull[attnum - 1]; return slot->tts_values[attnum - 1]; @@ -1102,7 +1644,7 @@ slot_getattr(TupleTableSlot *slot, int attnum, bool *isnull) /* * return NULL if attnum is out of range according to the tupdesc */ - if (attnum > tupleDesc->natts) + if (phynum > tupleDesc->nphyatts) { *isnull = true; return (Datum) 0; @@ -1123,7 +1665,7 @@ slot_getattr(TupleTableSlot *slot, int attnum, bool *isnull) * than the tupdesc.) */ tup = tuple->t_data; - if (attnum > HeapTupleHeaderGetNatts(tup)) + if (phynum > HeapTupleHeaderGetNatts(tup)) { *isnull = true; return (Datum) 0; @@ -1132,7 +1674,7 @@ slot_getattr(TupleTableSlot *slot, int attnum, bool *isnull) /* * check if target attribute is null: no point in groveling through tuple */ - if (HeapTupleHasNulls(tuple) && att_isnull(attnum - 1, tup->t_bits)) + if (HeapTupleHasNulls(tuple) && att_isnull(phynum - 1, tup->t_bits)) { *isnull = true; return (Datum) 0; @@ -1152,7 +1694,7 @@ slot_getattr(TupleTableSlot *slot, int attnum, bool *isnull) /* * Extract the attribute, along with any preceding attributes. */ - slot_deform_tuple(slot, attnum); + slot_deform_tuple(slot, phynum); /* * The result is acquired from tts_values array. @@ -1171,7 +1713,8 @@ void slot_getallattrs(TupleTableSlot *slot) { int tdesc_natts = slot->tts_tupleDescriptor->natts; - int attnum; + TupleDesc tupDesc = slot->tts_tupleDescriptor; + int phynum; HeapTuple tuple; /* Quick out if we have 'em all already */ @@ -1189,19 +1732,24 @@ slot_getallattrs(TupleTableSlot *slot) /* * load up any slots available from physical tuple */ - attnum = HeapTupleHeaderGetNatts(tuple->t_data); - attnum = Min(attnum, tdesc_natts); + phynum = HeapTupleHeaderGetNatts(tuple->t_data); + phynum = Min(phynum, tdesc_natts); - slot_deform_tuple(slot, attnum); + slot_deform_tuple(slot, phynum); /* * If tuple doesn't have all the atts indicated by tupleDesc, read the * rest as null */ - for (; attnum < tdesc_natts; attnum++) + for (; phynum < tdesc_natts; phynum++) { - slot->tts_values[attnum] = (Datum) 0; - slot->tts_isnull[attnum] = true; + AttrNumber attnum = tupDesc->attrmap[phynum]; + + if (attnum == InvalidAttrNumber) + continue; + + slot->tts_values[attnum - 1] = (Datum) 0; + slot->tts_isnull[attnum - 1] = true; } slot->tts_nvalid = tdesc_natts; } @@ -1212,18 +1760,19 @@ slot_getallattrs(TupleTableSlot *slot) * arrays to be valid at least up through the attnum'th entry. */ void -slot_getsomeattrs(TupleTableSlot *slot, int attnum) +slot_getsomeattrs(TupleTableSlot *slot, int phynum) { HeapTuple tuple; - int attno; + TupleDesc tupDesc = slot->tts_tupleDescriptor; + int phyno; /* Quick out if we have 'em all already */ - if (slot->tts_nvalid >= attnum) + if (slot->tts_nvalid >= phynum) return; /* Check for caller error */ - if (attnum <= 0 || attnum > slot->tts_tupleDescriptor->natts) - elog(ERROR, "invalid attribute number %d", attnum); + if (phynum <= 0 || phynum > slot->tts_tupleDescriptor->natts) + elog(ERROR, "invalid physical attribute number %d", phynum); /* * otherwise we had better have a physical tuple (tts_nvalid should equal @@ -1236,21 +1785,26 @@ slot_getsomeattrs(TupleTableSlot *slot, int attnum) /* * load up any slots available from physical tuple */ - attno = HeapTupleHeaderGetNatts(tuple->t_data); - attno = Min(attno, attnum); + phyno = HeapTupleHeaderGetNatts(tuple->t_data); + phyno = Min(phyno, phynum); - slot_deform_tuple(slot, attno); + slot_deform_tuple(slot, phyno); /* * If tuple doesn't have all the atts indicated by tupleDesc, read the * rest as null */ - for (; attno < attnum; attno++) + for (; phyno < phynum; phyno++) { - slot->tts_values[attno] = (Datum) 0; - slot->tts_isnull[attno] = true; + AttrNumber attno = tupDesc->attrmap[phyno]; + + if (attno == InvalidAttrNumber) + continue; + + slot->tts_values[attno - 1] = (Datum) 0; + slot->tts_isnull[attno - 1] = true; } - slot->tts_nvalid = attnum; + slot->tts_nvalid = phynum; } /* diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index 41d71c8..f48843f 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -20,6 +20,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "catalog/colstore.h" #include "catalog/pg_type.h" #include "miscadmin.h" #include "parser/parse_type.h" @@ -41,7 +42,9 @@ CreateTemplateTupleDesc(int natts, bool hasoid) { TupleDesc desc; char *stg; - int attroffset; + int attroffset1; + int attroffset2; + int attroffset3; /* * sanity checks @@ -49,8 +52,9 @@ CreateTemplateTupleDesc(int natts, bool hasoid) AssertArg(natts >= 0); /* - * Allocate enough memory for the tuple descriptor, including the - * attribute rows, and set up the attribute row pointers. + * Allocate enough memory for the tuple descriptor, including the attribute + * physical order to logical order map and attribute rows, and set up the + * attribute row pointers. * * Note: we assume that sizeof(struct tupleDesc) is a multiple of the * struct pointer alignment requirement, and hence we don't need to insert @@ -61,37 +65,54 @@ CreateTemplateTupleDesc(int natts, bool hasoid) * descriptors, so we only need ATTRIBUTE_FIXED_PART_SIZE space per attr. * That might need alignment padding, however. */ - attroffset = sizeof(struct tupleDesc) + natts * sizeof(Form_pg_attribute); - attroffset = MAXALIGN(attroffset); - stg = palloc(attroffset + natts * MAXALIGN(ATTRIBUTE_FIXED_PART_SIZE)); + + attroffset1 = MAXALIGN(sizeof(struct tupleDesc)); + attroffset2 = natts * sizeof(AttrNumber); + attroffset2 = MAXALIGN(attroffset2); + attroffset3 = natts * sizeof(Form_pg_attribute); + attroffset3 = MAXALIGN(attroffset3); + stg = palloc(attroffset1 + attroffset2 + attroffset3 + + natts * MAXALIGN(ATTRIBUTE_FIXED_PART_SIZE)); desc = (TupleDesc) stg; if (natts > 0) { + AttrNumber *attrmap; Form_pg_attribute *attrs; int i; - attrs = (Form_pg_attribute *) (stg + sizeof(struct tupleDesc)); - desc->attrs = attrs; - stg += attroffset; + stg += attroffset1; + attrmap = (AttrNumber *) stg; + stg += attroffset2; + attrs = (Form_pg_attribute *) stg; + stg += attroffset3; + for (i = 0; i < natts; i++) { attrs[i] = (Form_pg_attribute) stg; + attrmap[i] = i + 1; // is this necessary? stg += MAXALIGN(ATTRIBUTE_FIXED_PART_SIZE); } + desc->attrmap = attrmap; + desc->attrs = attrs; } else + { + desc->attrmap = NULL; desc->attrs = NULL; + } /* * Initialize other fields of the tupdesc. */ desc->natts = natts; + desc->nphyatts = natts; desc->constr = NULL; desc->tdtypeid = RECORDOID; desc->tdtypmod = -1; desc->tdhasoid = hasoid; desc->tdrefcount = -1; /* assume not reference-counted */ + desc->tdattorder = ATTRORDER_PHYSMATCHLOGICAL; return desc; } @@ -111,6 +132,7 @@ TupleDesc CreateTupleDesc(int natts, bool hasoid, Form_pg_attribute *attrs) { TupleDesc desc; + int i; /* * sanity checks @@ -118,14 +140,23 @@ CreateTupleDesc(int natts, bool hasoid, Form_pg_attribute *attrs) AssertArg(natts >= 0); desc = (TupleDesc) palloc(sizeof(struct tupleDesc)); + desc->attrmap = (AttrNumber *) palloc(natts * sizeof(AttrNumber)); desc->attrs = attrs; desc->natts = natts; + desc->nphyatts = natts; desc->constr = NULL; desc->tdtypeid = RECORDOID; desc->tdtypmod = -1; desc->tdhasoid = hasoid; desc->tdrefcount = -1; /* assume not reference-counted */ + /* XXX is this really necessary, or a good idea? */ + for (i = 0; i < natts; i++) + { + Assert(attrs[i]->attphynum <= natts); + desc->attrmap[attrs[i]->attphynum - 1] = i + 1; + } + return desc; } @@ -150,9 +181,11 @@ CreateTupleDescCopy(TupleDesc tupdesc) desc->attrs[i]->attnotnull = false; desc->attrs[i]->atthasdef = false; } + memcpy(desc->attrmap, tupdesc->attrmap, tupdesc->natts * sizeof(AttrNumber)); desc->tdtypeid = tupdesc->tdtypeid; desc->tdtypmod = tupdesc->tdtypmod; + desc->nphyatts = tupdesc->nphyatts; return desc; } @@ -175,6 +208,7 @@ CreateTupleDescCopyConstr(TupleDesc tupdesc) { memcpy(desc->attrs[i], tupdesc->attrs[i], ATTRIBUTE_FIXED_PART_SIZE); } + memcpy(desc->attrmap, tupdesc->attrmap, tupdesc->natts * sizeof(AttrNumber)); if (constr) { @@ -256,6 +290,9 @@ TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno, /* since we're not copying constraints or defaults, clear these */ dst->attrs[dstAttno - 1]->attnotnull = false; dst->attrs[dstAttno - 1]->atthasdef = false; + + /* Update the attrmap */ + dst->attrmap[dstAttno - 1] = dstAttno; } /* @@ -386,6 +423,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) return false; if (attr1->attlen != attr2->attlen) return false; + if (attr1->attphynum != attr2->attphynum) + return false; if (attr1->attndims != attr2->attndims) return false; if (attr1->atttypmod != attr2->atttypmod) @@ -474,6 +513,20 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) } /* + * TupleDescCacheReset + * Resets cached offsets of TupleDesc back to their initial values. + */ +void +TupleDescCacheReset(TupleDesc desc) +{ + int natts = desc->natts; + int i; + + for (i = 0; i < natts; i++) + desc->attrs[i]->attcacheoff = -1; +} + +/* * TupleDescInitEntry * This function initializes a single attribute structure in * a previously allocated tuple descriptor. @@ -529,6 +582,7 @@ TupleDescInitEntry(TupleDesc desc, att->atttypmod = typmod; att->attnum = attributeNumber; + att->attphynum = attributeNumber; att->attndims = attdim; att->attnotnull = false; @@ -580,12 +634,17 @@ TupleDescInitEntryCollation(TupleDesc desc, * * Given a relation schema (list of ColumnDef nodes), build a TupleDesc. * + * Also append ColumnStoreClauseInfo structs in *colstores, with one element for + * each attribute that contains a COLUMN STORE clause. colstores may be NULL if + * the caller is certain that there are no colstore clauses (e.g. when defining a + * view.) + * * Note: the default assumption is no OIDs; caller may modify the returned * TupleDesc if it wants OIDs. Also, tdtypeid will need to be filled in * later on. */ TupleDesc -BuildDescForRelation(List *schema) +BuildDescForRelation(List *schema, List **colstores) { int natts; AttrNumber attnum; @@ -648,6 +707,18 @@ BuildDescForRelation(List *schema) has_not_null |= entry->is_not_null; desc->attrs[attnum - 1]->attislocal = entry->is_local; desc->attrs[attnum - 1]->attinhcount = entry->inhcount; + + /* Fill in the column store info, if this column requires it */ + if (entry->cstoreClause) + { + ColumnStoreClauseInfo *store = palloc(sizeof(ColumnStoreClauseInfo)); + + store->attnum = attnum; + store->cstoreClause = entry->cstoreClause; + store->attnums = NIL; + store->cstoreOid = InvalidOid; + *colstores = lappend(*colstores, store); + } } if (has_not_null) diff --git a/src/backend/access/hash/hashfunc.c b/src/backend/access/hash/hashfunc.c index 9ee654e..608561a 100644 --- a/src/backend/access/hash/hashfunc.c +++ b/src/backend/access/hash/hashfunc.c @@ -74,6 +74,12 @@ hashoid(PG_FUNCTION_ARGS) return hash_uint32((uint32) PG_GETARG_OID(0)); } +Datum hashtid(PG_FUNCTION_ARGS) +{ + return hash_any((unsigned char *) PG_GETARG_POINTER(0), + sizeof(ItemPointerData)); +} + Datum hashenum(PG_FUNCTION_ARGS) { diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 3c2878b..fa1cd814 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1064,7 +1064,8 @@ fastgetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc, ) : ( - att_isnull((attnum) - 1, (tup)->t_data->t_bits) ? + att_isnull((tupleDesc)->attrs[(attnum) - 1]->attphynum - 1, + (tup)->t_data->t_bits) ? ( (*(isnull) = true), (Datum) NULL @@ -3591,6 +3592,15 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, } /* + * FIXME If there are any column stores on the table, we'll just disable + * HOT for now. This is effectively wrong and rather awful - we can + * do the HOT if no columns from column stores are updated, so we + * should fix this eventually. + */ + if (relation->rd_rel->relhascstore > 0) + satisfies_hot = false; + + /* * Note: beyond this point, use oldtup not otid to refer to old tuple. * otid may very well point at newtup->t_self, which we will overwrite * with the new tuple's location, so there's great risk of confusion if we diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index d8d1b06..10d38d2 100644 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -240,6 +240,7 @@ Boot_CreateStmt: BOOTSTRAP_SUPERUSERID, tupdesc, NIL, + NIL, RELKIND_RELATION, RELPERSISTENCE_PERMANENT, shared_relation, diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 95d6c14..4cd46ec 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -669,6 +669,7 @@ DefineAttr(char *name, char *type, int attnum, int nullness) namestrcpy(&attrtypes[attnum]->attname, name); elog(DEBUG4, "column %s %s", NameStr(attrtypes[attnum]->attname), type); attrtypes[attnum]->attnum = attnum + 1; /* fillatt */ + attrtypes[attnum]->attphynum = attnum + 1; typeoid = gettype(type); diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 25130ec..f28dd8c 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -10,9 +10,9 @@ subdir = src/backend/catalog top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -OBJS = catalog.o dependency.o heap.o index.o indexing.o namespace.o aclchk.o \ - objectaccess.o objectaddress.o pg_aggregate.o pg_collation.o \ - pg_constraint.o pg_conversion.o \ +OBJS = aclchk.o catalog.o colstore.o dependency.o heap.o index.o indexing.o \ + namespace.o objectaccess.o objectaddress.o \ + pg_aggregate.o pg_collation.o pg_constraint.o pg_conversion.o \ pg_depend.o pg_enum.o pg_inherits.o pg_largeobject.o pg_namespace.o \ pg_operator.o pg_proc.o pg_range.o pg_db_role_setting.o pg_shdepend.o \ pg_type.o storage.o toasting.o @@ -41,7 +41,7 @@ POSTGRES_BKI_SRCS = $(addprefix $(top_srcdir)/src/include/catalog/,\ pg_foreign_data_wrapper.h pg_foreign_server.h pg_user_mapping.h \ pg_foreign_table.h pg_policy.h pg_replication_origin.h \ pg_default_acl.h pg_seclabel.h pg_shseclabel.h \ - pg_collation.h pg_range.h pg_transform.h \ + pg_collation.h pg_range.h pg_transform.h pg_cstore_am.h pg_cstore.h \ toasting.h indexing.h \ ) diff --git a/src/backend/catalog/colstore.c b/src/backend/catalog/colstore.c new file mode 100644 index 0000000..1d3da06 --- /dev/null +++ b/src/backend/catalog/colstore.c @@ -0,0 +1,868 @@ +/*------------------------------------------------------------------------- + * + * colstore.c + * POSTGRES column store support code + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/backend/catalog/colstore.c + */ +#include "postgres.h" + +#include "access/genam.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/sysattr.h" +#include "catalog/catalog.h" +#include "catalog/colstore.h" +#include "catalog/dependency.h" +#include "catalog/heap.h" +#include "catalog/indexing.h" +#include "catalog/pg_cstore.h" +#include "catalog/pg_cstore_am.h" +#include "catalog/pg_namespace.h" +#include "catalog/pg_tablespace.h" /* GLOBALTABLESPACE_OID */ +#include "colstore/colstoreapi.h" +#include "commands/tablespace.h" +#include "miscadmin.h" +#include "nodes/bitmapset.h" +#include "nodes/parsenodes.h" +#include "nodes/execnodes.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/syscache.h" +#include "utils/rel.h" + +typedef struct ColumnStoreElem +{ + char *name; + Oid cst_am_oid; + Oid tablespaceId; + List *columns; /* of AttrNumber */ +} ColumnStoreElem; + +static void SequenceHeapAttributePhynum(TupleDesc tupdesc, Bitmapset *csattrs); +static TupleDesc ColumnStoreBuildDesc(ColumnStoreElem *elem, Relation parent); +static void AddNewColstoreTuple(Relation pg_cstore, ColumnStoreElem *entry, + Oid storeId, Oid relid); + +/* + * Figure out the set of column stores to create. + * + * decl_cstores is a list of column stores directly declared for the relation. + * inh_cstores is a list of column stores inherited from parent relations. + * We need this distinction because multiple uses of a column in declared ones + * is an error, but we ignore duplicates for the inherited ones. + * + * tablespaceId is the tablespace of the owning relation; it is used as the + * default tablespace for column stores that do not have a tablespace + * specification. + * + * Return value is a list of ColumnStoreElem. + */ +List * +DetermineColumnStores(TupleDesc tupdesc, List *decl_cstores, + List *inh_cstores, Oid tablespaceId) +{ + List *newstores = NIL; + Bitmapset *used; + ListCell *cell, + *cell2; + + /* + * There is no provision (not here, nor in the grammar) for a column that's + * part of a column store in the parent, but in the heap for the child + * table. Do we need a fix for that? + */ + + /* + * We use this bitmapset to keep track of columns which have already been + * assigned to a store. + */ + used = NULL; + + /* + * First scan the list of declared column stores. Using columns here more + * than once causes an error to be raised. + */ + foreach(cell, decl_cstores) + { + ColumnStoreClauseInfo *info = (ColumnStoreClauseInfo *) lfirst(cell); + ColumnStoreClause *clause = info->cstoreClause; + ColumnStoreElem *newstore; + + Assert(clause != NULL); + Assert((info->attnum != InvalidAttrNumber) || + (clause->columns != NIL)); + Assert(info->attnums == NIL && info->cstoreOid == InvalidOid); + + /* + * Verify that the name has not already been taken + */ + foreach(cell2, newstores) + { + ColumnStoreElem *elem = (ColumnStoreElem *) lfirst(cell2); + + if (strcmp(elem->name, clause->name) == 0) + /* XXX ereport */ + elog(ERROR, "duplicate column store name \"%s\"", clause->name); + } + + newstore = (ColumnStoreElem *) palloc(sizeof(ColumnStoreElem)); + newstore->name = clause->name; + newstore->cst_am_oid = GetColumnStoreAMByName(clause->storetype, false); + + /* + * Select tablespace to use. If not specified, use the tablespace + * of the parent relation. + * + * These are effectively the same checks as in DefineRelation. + */ + if (clause->tablespacename) + { + newstore->tablespaceId = + get_tablespace_oid(clause->tablespacename, false); + } + else + /* use tablespace of the relation */ + newstore->tablespaceId = tablespaceId; + + /* Check permissions except when using database's default */ + if (OidIsValid(newstore->tablespaceId) && + newstore->tablespaceId != MyDatabaseTableSpace) + { + AclResult aclresult; + + aclresult = pg_tablespace_aclcheck(newstore->tablespaceId, + GetUserId(), ACL_CREATE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, ACL_KIND_TABLESPACE, + get_tablespace_name(newstore->tablespaceId)); + } + + /* In all cases disallow placing user objects in pg_global */ + if (newstore->tablespaceId == GLOBALTABLESPACE_OID) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("only shared relations can be placed in pg_global tablespace"))); + + /* + * Fill in the attnum list: if it's a column store declared as column + * constraint, the only attnum was already determined by + * BuildDescForRelation. Otherwise we need to resolve column names to + * attnums using the tuple descriptor. + */ + if (info->attnum != InvalidAttrNumber) + newstore->columns = list_make1_int(info->attnum); + else + { + newstore->columns = NIL; + + foreach(cell2, clause->columns) + { + char *colname = strVal(lfirst(cell2)); + AttrNumber attnum, + i; + + attnum = InvalidAttrNumber; + for (i = 1; i <= tupdesc->natts; i++) + { + if (namestrcmp(&(tupdesc->attrs[i - 1]->attname), colname) == 0) + attnum = i; + } + /* XXX ereport */ + if (attnum == InvalidAttrNumber) + elog(ERROR, "no column \"%s\" in the table", colname); + + newstore->columns = lappend_int(newstore->columns, attnum); + } + } + + /* Make sure there are no columns specified in multiple stores */ + foreach(cell2, newstore->columns) + { + AttrNumber attno = lfirst_int(cell2); + + /* XXX ereport */ + if (bms_is_member(attno, used)) + elog(ERROR, "column already in a store"); + + used = bms_add_member(used, attno); + } + + newstores = lappend(newstores, newstore); + } + + /* + * Now process the list of column stores coming from parent relations. + * Columns that are already used by previous stores are silently ignored. + * (In particular, this means that some parent stores might not exist at + * all in the child). + */ + foreach (cell, inh_cstores) + { + ColumnStoreClauseInfo *info = (ColumnStoreClauseInfo *) lfirst(cell); + Relation parentstore; + List *attnums = NIL; + + Assert((info->attnum == InvalidAttrNumber) && + (info->cstoreClause == NULL)); + Assert((info->attnums != NIL) && + (info->cstoreOid != InvalidOid)); + + parentstore = relation_open(info->cstoreOid, AccessShareLock); + + /* + * Examine the column list first. If all columns are used in + * previously defined column stores, we can ignore this one. + */ + foreach(cell2, info->attnums) + { + AttrNumber attnum = lfirst_int(cell2); + + if (bms_is_member(attnum, used)) + continue; + attnums = lappend_int(attnums, attnum); + used = bms_add_member(used, attnum); + } + + /* If we ended up with a nonempty list, add this store to the list */ + if (attnums != NIL) + { + ColumnStoreElem *newstore; + newstore = (ColumnStoreElem *) palloc(sizeof(ColumnStoreElem)); + + newstore->name = pstrdup(NameStr(parentstore->rd_cstore->cstname)); + newstore->cst_am_oid = parentstore->rd_rel->relam; + newstore->tablespaceId = parentstore->rd_rel->reltablespace; + newstore->columns = attnums; + + newstores = lappend(newstores, newstore); + } + + relation_close(parentstore, AccessShareLock); + } + + /* + * Check that the names of the column stores are unique, so that we can + * print a nice error message instead of a confusing unique violation + * later. This is O(N^2), but that should not be a problem. + * + * XXX We don't have relname here, so we can't put it to the message. + */ + foreach (cell, newstores) + { + ListCell *cell2; + ColumnStoreElem *elem1 = (ColumnStoreElem *) lfirst(cell); + + foreach (cell2, newstores) + { + ColumnStoreElem *elem2 = (ColumnStoreElem *) lfirst(cell2); + + if (elem1 == elem2) /* skip the same element */ + continue; + + /* same names */ + if (strcmp(elem1->name, elem2->name) == 0) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("column store \"%s\" already exists", elem1->name))); + + } + } + + SequenceHeapAttributePhynum(tupdesc, used); + + /* + * Return the info we collected. + */ + return newstores; +} + +static void +SequenceHeapAttributePhynum(TupleDesc tupdesc, Bitmapset *csattrs) +{ + int numatts; + int i; + int attphynum; + + numatts = tupdesc->natts; + + attphynum = 0; + + for (i = 0; i < numatts; i++) + { + if (bms_is_member(i + 1, csattrs)) + { + tupdesc->attrs[i]->attphynum = InvalidAttrNumber; + continue; + } + + tupdesc->attrs[i]->attphynum = ++attphynum; + } + + /* + * Initialize the map which translates the phyiscal attnum back into the + * logical attnum + */ + for (i = 0; i < numatts; i++) + { + AttrNumber phynum = tupdesc->attrs[i]->attphynum; + + if (phynum != InvalidAttrNumber) + tupdesc->attrmap[phynum - 1] = i + 1; + } + + tupdesc->nphyatts = attphynum; +} + +/* Create the column stores for the given table. This creates the files + * assigned as relfilenode for each column store; also, the pg_class and + * pg_cstore catalog entries are created. + */ +void +CreateColumnStores(Relation rel, List *colstores) +{ + Relation pg_class; + Relation pg_cstore; + ListCell *cell; + int storenum = 1; + ObjectAddress parentrel; + + if (colstores == NIL) + return; + + pg_class = heap_open(RelationRelationId, RowExclusiveLock); + pg_cstore = heap_open(CStoreRelationId, RowExclusiveLock); + + ObjectAddressSet(parentrel, RelationRelationId, + RelationGetRelid(rel)); + + foreach(cell, colstores) + { + ColumnStoreElem *elem = (ColumnStoreElem *) lfirst(cell); + Relation store; + Oid newStoreId; + TupleDesc storedesc = ColumnStoreBuildDesc(elem, rel); + char *classname; + ObjectAddress myself; + + /* + * Get the OID for the new column store. + */ + newStoreId = GetNewRelFileNode(elem->tablespaceId, pg_class, + rel->rd_rel->relpersistence); + + classname = psprintf("pg_colstore_%u_%d", + RelationGetRelid(rel), storenum++); + + /* + * Create the relcache entry for the store. This also creates the + * underlying storage; it's smgr's responsibility to remove the file if + * we fail later on. + */ + store = + heap_create(classname, + PG_COLSTORE_NAMESPACE, + elem->tablespaceId, + newStoreId, + InvalidOid, + storedesc, + RELKIND_COLUMN_STORE, + rel->rd_rel->relpersistence, + false, + false, + allowSystemTableMods); + + /* insert pg_attribute tuples */ + AddNewAttributeTuples(newStoreId, + storedesc, + RELKIND_COLUMN_STORE, + false, + 0); + + /* + * Insert the pg_class tuple + */ + store->rd_rel->relam = elem->cst_am_oid; + InsertPgClassTuple(pg_class, store, newStoreId, + (Datum) 0, (Datum) 0); + + /* And finally insert the pg_cstore tuple */ + AddNewColstoreTuple(pg_cstore, elem, newStoreId, + RelationGetRelid(rel)); + + /* + * This is a good place to record dependencies. We choose to have all the + * subsidiary entries (both pg_class and pg_cstore entries) depend on the + * pg_class entry for the main relation. + */ + ObjectAddressSet(myself, CStoreRelationId, newStoreId); + recordDependencyOn(&myself, &parentrel, DEPENDENCY_INTERNAL); + ObjectAddressSet(myself, RelationRelationId, newStoreId); + recordDependencyOn(&myself, &parentrel, DEPENDENCY_INTERNAL); + + heap_close(store, NoLock); + } + + heap_close(pg_class, RowExclusiveLock); + heap_close(pg_cstore, RowExclusiveLock); +} + +#include "catalog/pg_type.h" /* XXX for TIDOID below. Remove */ + +/* + * Build a tuple descriptor for a not-yet-catalogued column store. + * + * XXX ugly hack: consider column 1 to carry heap TID. Remove someday + */ +static TupleDesc +ColumnStoreBuildDesc(ColumnStoreElem *elem, Relation parent) +{ + TupleDesc tupdesc; + TupleDesc parentdesc; + ListCell *cell; + AttrNumber attnum = 1; + + parentdesc = RelationGetDescr(parent); + + tupdesc = CreateTemplateTupleDesc(list_length(elem->columns) + 1, false); + + /* the heap TID is first column */ + TupleDescInitEntry(tupdesc, attnum++, "heaptid", TIDOID, -1, 0); + + foreach(cell, elem->columns) + { + AttrNumber parentattnum = lfirst_int(cell); + + TupleDescInitEntry(tupdesc, + attnum++, + NameStr(parentdesc->attrs[parentattnum - 1]->attname), + parentdesc->attrs[parentattnum - 1]->atttypid, + parentdesc->attrs[parentattnum - 1]->atttypmod, + parentdesc->attrs[parentattnum - 1]->attndims); + } + + return tupdesc; +} + +/* + * Return a list of column store definitions for an existing table + */ +List * +CloneColumnStores(Relation rel) +{ + /* FIXME fill this in */ + return NIL; +} + +/* + * Add a new pg_cstore tuple for a column store + */ +static void +AddNewColstoreTuple(Relation pg_cstore, ColumnStoreElem *entry, + Oid storeId, Oid relid) +{ + HeapTuple newtup; + ListCell *cell; + Datum values[Natts_pg_cstore]; + bool nulls[Natts_pg_cstore]; + NameData cstname; + int natts; + int16 *attrarr; + int2vector *attrs; + int i = 0; + + /* build the int2vector of attribute numbers */ + natts = list_length(entry->columns); + Assert(natts > 0); + attrarr = palloc(sizeof(int16 *) * natts); + foreach(cell, entry->columns) + attrarr[i++] = (AttrNumber) lfirst_int(cell); + attrs = buildint2vector(attrarr, natts); + + /* build the pg_cstore tuple */ + namestrcpy(&cstname, entry->name); + values[Anum_pg_cstore_cstname - 1] = NameGetDatum(&cstname); + values[Anum_pg_cstore_cstrelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_cstore_cststoreid - 1] = ObjectIdGetDatum(storeId); + values[Anum_pg_cstore_cstnatts - 1] = Int32GetDatum(list_length(entry->columns)); + values[Anum_pg_cstore_cstatts - 1] = PointerGetDatum(attrs); + memset(nulls, 0, sizeof(nulls)); + newtup = heap_form_tuple(RelationGetDescr(pg_cstore), values, nulls); + + HeapTupleSetOid(newtup, storeId); + + /* insert it into pg_cstore */ + simple_heap_insert(pg_cstore, newtup); + + /* keep indexes current */ + CatalogUpdateIndexes(pg_cstore, newtup); + + heap_freetuple(newtup); +} + +Oid +get_relation_cstore_oid(Oid relid, const char *cstore_name, bool missing_ok) +{ + Relation pg_cstore_rel; + ScanKeyData skey[2]; + SysScanDesc sscan; + HeapTuple cstore_tuple; + Oid cstore_oid; + + pg_cstore_rel = heap_open(CStoreRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], + Anum_pg_cstore_cstrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&skey[1], + Anum_pg_cstore_cstname, + BTEqualStrategyNumber, F_NAMEEQ, + CStringGetDatum(cstore_name)); + + sscan = systable_beginscan(pg_cstore_rel, + CStoreCstRelidCstnameIndexId, true, NULL, 2, + skey); + + cstore_tuple = systable_getnext(sscan); + + if (!HeapTupleIsValid(cstore_tuple)) + { + if (!missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("column store \"%s\" for table \"%s\" does not exist", + cstore_name, get_rel_name(relid)))); + + cstore_oid = InvalidOid; + } + else + cstore_oid = HeapTupleGetOid(cstore_tuple); + + /* Clean up. */ + systable_endscan(sscan); + heap_close(pg_cstore_rel, AccessShareLock); + + return cstore_oid; +} + +void +RemoveColstoreById(Oid cstoreOid) +{ + Relation pg_cstore; + HeapTuple tuple; + + pg_cstore = heap_open(CStoreRelationId, RowExclusiveLock); + + tuple = SearchSysCache1(CSTOREOID, ObjectIdGetDatum(cstoreOid)); + + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for column store %u", cstoreOid); + + simple_heap_delete(pg_cstore, &tuple->t_self); + + ReleaseSysCache(tuple); + + heap_close(pg_cstore, RowExclusiveLock); +} + +Oid +GetColumnStoreAMByName(char *cstamname, bool missing_ok) +{ + Oid oid; + + oid = GetSysCacheOid1(CSTOREAMNAME, + CStringGetDatum(cstamname)); + if (!OidIsValid(oid)) + { + if (missing_ok) + return InvalidOid; + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("column store access method \"%s\" does not exist", + cstamname))); + } + + return oid; +} + +/* + * BuildColumnStoreHandler + * Construct a ColumnStoreHandler to operate a column store. This + * also calls the Open routine. + * + * XXX why not have this function take an OID instead of a Relation? + */ +ColumnStoreHandler * +BuildColumnStoreHandler(Relation cstore, bool for_write, LOCKMODE lockmode, + Snapshot snapshot) +{ + ColumnStoreHandler *cs = makeNode(ColumnStoreHandler); + Form_pg_cstore cstoreStruct = cstore->rd_cstore; + int i; + int numKeys; + + cs->csh_relationDesc = cstore; + cs->csh_lockmode = lockmode; + /* opaque is set by Open, below */ + + /* Obtain and cache the ColumnStoreRoutine pointer for this colstore */ + cs->csh_ColumnStoreRoutine = + GetColumnStoreRoutineForRelation(cstore, true); + + /* check the number of keys, and copy attr numbers into the IndexInfo */ + numKeys = cstoreStruct->cstnatts; + if (numKeys < 1 || numKeys > INDEX_MAX_KEYS) + elog(ERROR, "invalid cstnatts %d for column store %u", + numKeys, RelationGetRelid(cstore)); + cs->csh_NumColumnStoreAttrs = numKeys; + for (i = 0; i < numKeys; i++) + cs->csh_KeyAttrNumbers[i] = cstoreStruct->cstatts.values[i]; + + /* fill in the opaque pointer */ + cs->csh_ColumnStoreRoutine->ExecColumnStoreOpen(cs, for_write, snapshot); + + return cs; +} + +/* + * Free all resources associated with a column store handler + */ +void +CloseColumnStore(ColumnStoreHandler *csthandler) +{ + csthandler->csh_ColumnStoreRoutine->ExecColumnStoreClose(csthandler); + + heap_close(csthandler->csh_relationDesc, csthandler->csh_lockmode); + + pfree(csthandler); +} + +/* ---------------- + * FormColumnStoreDatum + * Construct values[] and isnull[] arrays for a new column store tuple. + * + * handler column store handler + * slot Heap tuple for which we must prepare a column store entry + * values Array of column store Datums (output area) + * isnull Array of is-null indicators (output area) + * ---------------- + */ +void +FormColumnStoreDatum(ColumnStoreHandler *handler, + HeapTuple tuple, + TupleDesc tupdesc, + Datum *values, + bool *isnull) +{ + int i; + + /* + * XXX nasty hack: inject the TID as first column. This should be + * parametrized better somehow ... + */ + values[0] = PointerGetDatum(&(tuple->t_self)); /* XXX should be ItemPointerGetDatum */ + isnull[0] = false; + + for (i = 1; i <= handler->csh_NumColumnStoreAttrs; i++) + values[i] = heap_getlogattr(tuple, + handler->csh_KeyAttrNumbers[i - 1], + tupdesc, + &isnull[i]); +} + +/* + * Builds a descriptor for the heap part of the relation, and a tuple with only + * the relevant attributes. + */ +HeapTuple +FilterHeapTuple(ResultRelInfo *resultRelInfo, HeapTuple tuple, TupleDesc *heapdesc) +{ + int attnum; + Bitmapset *cstoreatts = NULL; /* attributes mentioned in colstore */ + TupleDesc origdesc = resultRelInfo->ri_RelationDesc->rd_att; + HeapTuple newtup; + ListCell *l; + + /* used to build the new descriptor / tuple */ + int natts; + int i; + Datum *values; + bool *nulls; + + /* should not be called with no column stores */ + Assert(resultRelInfo->ri_ColumnStoreHandler != NIL); + + /* XXX should use attinheap here instead of this */ + foreach(l, resultRelInfo->ri_ColumnStoreHandler) + { + ColumnStoreHandler *handler = lfirst(l); + int i; + + for (i = 0; i < handler->csh_NumColumnStoreAttrs; i++) + cstoreatts = bms_add_member(cstoreatts, + handler->csh_KeyAttrNumbers[i]); + } + + /* we should get some columns from column stores */ + Assert(bms_num_members(cstoreatts) > 0); + + /* the new descriptor contains only the remaining attributes */ + natts = origdesc->natts - bms_num_members(cstoreatts); + + *heapdesc = CreateTemplateTupleDesc(natts, false); + + values = (Datum *) palloc0(sizeof(Datum) * natts); + nulls = (bool *) palloc0(sizeof(bool) * natts); + + attnum = 1; + for (i = 0; i < origdesc->natts; i++) + { + /* if part of a column store, skip the attribute */ + if (bms_is_member(origdesc->attrs[i]->attnum, cstoreatts)) + continue; + + values[attnum - 1] = heap_getattr(tuple, i + 1, origdesc, + &nulls[attnum - 1]); + + TupleDescCopyEntry(*heapdesc, attnum++, origdesc, i + 1); + } + + newtup = heap_form_tuple(*heapdesc, values, nulls); + + /* copy important header fields */ + newtup->t_self = tuple->t_self; + newtup->t_data->t_ctid = tuple->t_data->t_ctid; + + pfree(values); + pfree(nulls); + + return newtup; +} + +/* + * GetColumnStoreRoutine - call the specified column store handler routine + * to get its ColumnStoreRoutine struct. + */ +ColumnStoreRoutine * +GetColumnStoreRoutine(Oid cstamhandler) +{ + Datum datum; + ColumnStoreRoutine *routine; + + datum = OidFunctionCall0(cstamhandler); + routine = (ColumnStoreRoutine *) DatumGetPointer(datum); + + if (routine == NULL || !IsA(routine, ColumnStoreRoutine)) + elog(ERROR, "column store handler function %u did not return an ColumnStoreRoutine struct", + cstamhandler); + + return routine; +} + +/* + * GetColumnStoreHandlerByRelId - look up the handler of the column store handler + * for the given column store relation + */ +Oid +GetColumnStoreHandlerByRelId(Oid relid) +{ + HeapTuple tp; + Form_pg_cstore_am cstform; + Oid cstamhandler; + Relation rel; + Oid amoid; + + rel = relation_open(relid, AccessShareLock); + amoid = rel->rd_rel->relam; + + Assert(amoid != InvalidOid); + Assert(rel->rd_rel->relkind == RELKIND_COLUMN_STORE); + + relation_close(rel, AccessShareLock); + + /* Get server OID for the foreign table. */ + tp = SearchSysCache1(CSTOREAMOID, ObjectIdGetDatum(amoid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for foreign table %u", amoid); + cstform = (Form_pg_cstore_am) GETSTRUCT(tp); + cstamhandler = cstform->cstamhandler; + + /* Complain if column store has been set to NO HANDLER. */ + if (!OidIsValid(cstamhandler)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("column store \"%s\" has no handler", + NameStr(cstform->cstamname)))); + + ReleaseSysCache(tp); + + return cstamhandler; +} + +/* + * GetColumnStoreRoutineByRelId - look up the handler of the column store, and + * retrieve its ColumnStoreRoutine struct. + */ +ColumnStoreRoutine * +GetColumnStoreRoutineByRelId(Oid relid) +{ + Oid cstamhandler = GetColumnStoreHandlerByRelId(relid); + + return GetColumnStoreRoutine(cstamhandler); +} + +/* + * GetColumnStoreRoutineForRelation - look up the handler of the given column + * store, and retrieve its ColumnStoreRoutine struct. + * + * This function is preferred over GetColumnStoreRoutineByRelId because it + * caches the data in the relcache entry, saving a number of catalog lookups. + * + * If makecopy is true then the returned data is freshly palloc'd in the + * caller's memory context. Otherwise, it's a pointer to the relcache data, + * which will be lost in any relcache reset --- so don't rely on it long. + */ +ColumnStoreRoutine * +GetColumnStoreRoutineForRelation(Relation relation, bool makecopy) +{ + ColumnStoreRoutine *cstroutine; + ColumnStoreRoutine *ccstroutine; + + Assert(relation->rd_rel->relkind == RELKIND_COLUMN_STORE); + Assert(relation->rd_rel->relam != 0); + + if (relation->rd_colstoreroutine == NULL) + { + /* Get the info by consulting the catalogs */ + cstroutine = GetColumnStoreRoutineByRelId(RelationGetRelid(relation)); + + /* Save the data for later reuse in CacheMemoryContext */ + ccstroutine = + (ColumnStoreRoutine *) MemoryContextAlloc(CacheMemoryContext, + sizeof(ColumnStoreRoutine)); + memcpy(ccstroutine, cstroutine, sizeof(ColumnStoreRoutine)); + relation->rd_colstoreroutine = ccstroutine; + + /* Give back the locally palloc'd copy regardless of makecopy */ + return cstroutine; + } + + /* We have valid cached data --- does the caller want a copy? */ + if (makecopy) + { + ccstroutine = (ColumnStoreRoutine *) palloc(sizeof(ColumnStoreRoutine)); + memcpy(ccstroutine, relation->rd_colstoreroutine, sizeof(ColumnStoreRoutine)); + return ccstroutine; + } + + /* Only a short-lived reference is needed, so just hand back cached copy */ + return relation->rd_colstoreroutine; +} diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index efca34c..5d76e97 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -16,6 +16,7 @@ #include "access/htup_details.h" #include "access/xact.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -144,6 +145,7 @@ static const Oid object_classes[] = { AccessMethodProcedureRelationId, /* OCLASS_AMPROC */ RewriteRelationId, /* OCLASS_REWRITE */ TriggerRelationId, /* OCLASS_TRIGGER */ + CStoreRelationId, /* OCLASS_COLSTORE */ NamespaceRelationId, /* OCLASS_SCHEMA */ TSParserRelationId, /* OCLASS_TSPARSER */ TSDictionaryRelationId, /* OCLASS_TSDICT */ @@ -1214,6 +1216,10 @@ doDeletion(const ObjectAddress *object, int flags) RemoveTriggerById(object->objectId); break; + case OCLASS_COLSTORE: + RemoveColstoreById(object->objectId); + break; + case OCLASS_SCHEMA: RemoveSchemaById(object->objectId); break; @@ -2412,6 +2418,9 @@ getObjectClass(const ObjectAddress *object) case PolicyRelationId: return OCLASS_POLICY; + case CStoreRelationId: + return OCLASS_COLSTORE; + case TransformRelationId: return OCLASS_TRANSFORM; } diff --git a/src/backend/catalog/genbki.pl b/src/backend/catalog/genbki.pl index d06eae0..3968820 100644 --- a/src/backend/catalog/genbki.pl +++ b/src/backend/catalog/genbki.pl @@ -217,6 +217,7 @@ foreach my $catname (@{ $catalogs->{names} }) $attnum++; my $row = emit_pgattr_row($table_name, $attr, $priornotnull); $row->{attnum} = $attnum; + $row->{attphynum} = $attnum; $row->{attstattarget} = '-1'; $priornotnull &= ($row->{attnotnull} eq 't'); @@ -254,6 +255,7 @@ foreach my $catname (@{ $catalogs->{names} }) $attnum--; my $row = emit_pgattr_row($table_name, $attr, 1); $row->{attnum} = $attnum; + $row->{attphynum} = $attnum; $row->{attstattarget} = '0'; # some catalogs don't have oids diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 04c4f8f..6ef53cf 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -37,6 +37,7 @@ #include "access/xlog.h" #include "catalog/binary_upgrade.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -55,6 +56,7 @@ #include "catalog/storage_xlog.h" #include "commands/tablecmds.h" #include "commands/typecmds.h" +#include "executor/executor.h" #include "miscadmin.h" #include "nodes/nodeFuncs.h" #include "optimizer/var.h" @@ -136,38 +138,38 @@ static List *insert_ordered_unique_oid(List *list, Oid datum); static FormData_pg_attribute a1 = { 0, {"ctid"}, TIDOID, 0, sizeof(ItemPointerData), - SelfItemPointerAttributeNumber, 0, -1, -1, - false, 'p', 's', true, false, false, true, 0 + SelfItemPointerAttributeNumber, SelfItemPointerAttributeNumber, 0, -1, + -1, false, 'p', 's', true, false, false, true, 0 }; static FormData_pg_attribute a2 = { 0, {"oid"}, OIDOID, 0, sizeof(Oid), - ObjectIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + ObjectIdAttributeNumber, ObjectIdAttributeNumber, 0, -1, + -1, true, 'p', 'i', true, false, false, true, 0 }; static FormData_pg_attribute a3 = { 0, {"xmin"}, XIDOID, 0, sizeof(TransactionId), - MinTransactionIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + MinTransactionIdAttributeNumber, MinTransactionIdAttributeNumber, 0, -1, + -1, true, 'p', 'i', true, false, false, true, 0 }; static FormData_pg_attribute a4 = { 0, {"cmin"}, CIDOID, 0, sizeof(CommandId), - MinCommandIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + MinCommandIdAttributeNumber, MinCommandIdAttributeNumber, 0, -1, + -1, true, 'p', 'i', true, false, false, true, 0 }; static FormData_pg_attribute a5 = { 0, {"xmax"}, XIDOID, 0, sizeof(TransactionId), - MaxTransactionIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + MaxTransactionIdAttributeNumber, MaxTransactionIdAttributeNumber, 0, -1, + -1, true, 'p', 'i', true, false, false, true, 0 }; static FormData_pg_attribute a6 = { 0, {"cmax"}, CIDOID, 0, sizeof(CommandId), - MaxCommandIdAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + MaxCommandIdAttributeNumber, MaxCommandIdAttributeNumber, 0, -1, + -1, true, 'p', 'i', true, false, false, true, 0 }; /* @@ -178,8 +180,8 @@ static FormData_pg_attribute a6 = { */ static FormData_pg_attribute a7 = { 0, {"tableoid"}, OIDOID, 0, sizeof(Oid), - TableOidAttributeNumber, 0, -1, -1, - true, 'p', 'i', true, false, false, true, 0 + TableOidAttributeNumber, TableOidAttributeNumber, 0, -1, + -1, true, 'p', 'i', true, false, false, true, 0 }; static const Form_pg_attribute SysAtt[] = {&a1, &a2, &a3, &a4, &a5, &a6, &a7}; @@ -615,6 +617,7 @@ InsertPgAttributeTuple(Relation pg_attribute_rel, values[Anum_pg_attribute_attstattarget - 1] = Int32GetDatum(new_attribute->attstattarget); values[Anum_pg_attribute_attlen - 1] = Int16GetDatum(new_attribute->attlen); values[Anum_pg_attribute_attnum - 1] = Int16GetDatum(new_attribute->attnum); + values[Anum_pg_attribute_attphynum - 1] = Int16GetDatum(new_attribute->attphynum); values[Anum_pg_attribute_attndims - 1] = Int32GetDatum(new_attribute->attndims); values[Anum_pg_attribute_attcacheoff - 1] = Int32GetDatum(new_attribute->attcacheoff); values[Anum_pg_attribute_atttypmod - 1] = Int32GetDatum(new_attribute->atttypmod); @@ -653,7 +656,7 @@ InsertPgAttributeTuple(Relation pg_attribute_rel, * tuples to pg_attribute. * -------------------------------- */ -static void +void AddNewAttributeTuples(Oid new_rel_oid, TupleDesc tupdesc, char relkind, @@ -712,10 +715,11 @@ AddNewAttributeTuples(Oid new_rel_oid, /* * Next we add the system attributes. Skip OID if rel has no OIDs. Skip - * all for a view or type relation. We don't bother with making datatype - * dependencies here, since presumably all these types are pinned. + * all for a colstore, view or type relation. We don't bother with making + * datatype dependencies here, since presumably all these types are pinned. */ - if (relkind != RELKIND_VIEW && relkind != RELKIND_COMPOSITE_TYPE) + if (relkind != RELKIND_VIEW && relkind != RELKIND_COMPOSITE_TYPE && + relkind != RELKIND_COLUMN_STORE) { for (i = 0; i < (int) lengthof(SysAtt); i++) { @@ -792,6 +796,7 @@ InsertPgClassTuple(Relation pg_class_desc, values[Anum_pg_class_relallvisible - 1] = Int32GetDatum(rd_rel->relallvisible); values[Anum_pg_class_reltoastrelid - 1] = ObjectIdGetDatum(rd_rel->reltoastrelid); values[Anum_pg_class_relhasindex - 1] = BoolGetDatum(rd_rel->relhasindex); + values[Anum_pg_class_relhascstore - 1] = BoolGetDatum(rd_rel->relhascstore); values[Anum_pg_class_relisshared - 1] = BoolGetDatum(rd_rel->relisshared); values[Anum_pg_class_relpersistence - 1] = CharGetDatum(rd_rel->relpersistence); values[Anum_pg_class_relkind - 1] = CharGetDatum(rd_rel->relkind); @@ -994,6 +999,7 @@ AddNewRelationType(const char *typeName, * ownerid: OID of new rel's owner * tupdesc: tuple descriptor (source of column definitions) * cooked_constraints: list of precooked check constraints and defaults + * colstores: list (of ColumnStoreElem) of column stores for this rel * relkind: relkind for new rel * relpersistence: rel's persistence status (permanent, temp, or unlogged) * shared_relation: TRUE if it's to be a shared relation @@ -1023,6 +1029,7 @@ heap_create_with_catalog(const char *relname, Oid ownerid, TupleDesc tupdesc, List *cooked_constraints, + List *colstores, char relkind, char relpersistence, bool shared_relation, @@ -1248,6 +1255,9 @@ heap_create_with_catalog(const char *relname, pfree(relarrayname); } + /* Set relhascstore correctly */ + new_rel_desc->rd_rel->relhascstore = colstores != NIL; + /* * now create an entry in pg_class for the relation. * @@ -1266,6 +1276,13 @@ heap_create_with_catalog(const char *relname, reloptions); /* + * If the new relation has any column stores, create them now. This + * assigns their OIDs and creates the files on disk (it's smgr's + * responsibility to remove these files if we fail below.) + */ + CreateColumnStores(new_rel_desc, colstores); + + /* * now add tuples to pg_attribute for the attributes in our new relation. */ AddNewAttributeTuples(relid, new_rel_desc->rd_att, relkind, @@ -2805,6 +2822,9 @@ heap_truncate_one_rel(Relation rel) /* keep the lock... */ heap_close(toastrel, NoLock); } + + if (rel->rd_rel->relhascstore) + ExecTruncateColumnStores(rel); } /* diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index c10be3d..f89594d 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -351,6 +351,7 @@ ConstructTupleDescriptor(Relation heapRelation, * attr */ to->attnum = i + 1; + to->attphynum = i + 1; to->attstattarget = -1; to->attcacheoff = -1; @@ -385,6 +386,7 @@ ConstructTupleDescriptor(Relation heapRelation, * Assign some of the attributes values. Leave the rest as 0. */ to->attnum = i + 1; + to->attphynum = i + 1; to->atttypid = keyType; to->attlen = typeTup->typlen; to->attbyval = typeTup->typbyval; diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index e44d7d0..3d9ffbe 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -18,6 +18,7 @@ #include "access/htup_details.h" #include "access/sysattr.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/indexing.h" #include "catalog/objectaddress.h" #include "catalog/pg_amop.h" @@ -132,6 +133,18 @@ static const ObjectPropertyType ObjectProperty[] = true }, { + CStoreRelationId, + CStoreOidIndexId, + CSTOREOID, + -1, + Anum_pg_cstore_cstname, + InvalidAttrNumber, + InvalidAttrNumber, + InvalidAttrNumber, + -1, + false + }, + { ConstraintRelationId, ConstraintOidIndexId, CONSTROID, @@ -575,6 +588,10 @@ static const struct object_type_map { "trigger", OBJECT_TRIGGER }, + /* OCLASS_COLSTORE */ + { + "column store", OBJECT_COLSTORE + }, /* OCLASS_SCHEMA */ { "schema", OBJECT_SCHEMA @@ -765,6 +782,7 @@ get_object_address(ObjectType objtype, List *objname, List *objargs, case OBJECT_TRIGGER: case OBJECT_TABCONSTRAINT: case OBJECT_POLICY: + case OBJECT_COLSTORE: address = get_object_address_relobject(objtype, objname, &relation, missing_ok); break; @@ -1278,6 +1296,13 @@ get_object_address_relobject(ObjectType objtype, List *objname, InvalidOid; address.objectSubId = 0; break; + case OBJECT_COLSTORE: + address.classId = CStoreRelationId; + address.objectId = relation ? + get_relation_cstore_oid(reloid, depname, missing_ok) : + InvalidOid; + address.objectSubId = 0; + break; default: elog(ERROR, "unrecognized objtype: %d", (int) objtype); /* placate compiler, which doesn't know elog won't return */ @@ -2028,6 +2053,7 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, case OBJECT_TRIGGER: case OBJECT_POLICY: case OBJECT_TABCONSTRAINT: + case OBJECT_COLSTORE: if (!pg_class_ownercheck(RelationGetRelid(relation), roleid)) aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, RelationGetRelationName(relation)); @@ -2791,6 +2817,30 @@ getObjectDescription(const ObjectAddress *object) break; } + case OCLASS_COLSTORE: + { + Relation cstDesc; + HeapTuple tup; + Form_pg_cstore store; + + cstDesc = heap_open(CStoreRelationId, AccessShareLock); + + tup = get_catalog_object_by_oid(cstDesc, object->objectId); + + if (!HeapTupleIsValid(tup)) + elog(ERROR, "could not find tuple for column store %u", + object->objectId); + + store = (Form_pg_cstore) GETSTRUCT(tup); + + appendStringInfo(&buffer, _("column store %s on "), + NameStr(store->cstname)); + getRelationDescription(&buffer, store->cstrelid); + + heap_close(cstDesc, AccessShareLock); + break; + } + case OCLASS_TRANSFORM: { HeapTuple trfTup; @@ -3544,6 +3594,10 @@ getObjectTypeDescription(const ObjectAddress *object) appendStringInfoString(&buffer, "trigger"); break; + case OCLASS_COLSTORE: + appendStringInfoString(&buffer, "column store"); + break; + case OCLASS_SCHEMA: appendStringInfoString(&buffer, "schema"); break; @@ -4185,6 +4239,32 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_COLSTORE: + { + Relation cstDesc; + HeapTuple tup; + Form_pg_cstore store; + + cstDesc = heap_open(CStoreRelationId, AccessShareLock); + + tup = get_catalog_object_by_oid(cstDesc, object->objectId); + + if (!HeapTupleIsValid(tup)) + elog(ERROR, "could not find tuple for column store %u", + object->objectId); + + store = (Form_pg_cstore) GETSTRUCT(tup); + + appendStringInfo(&buffer, "%s on ", + quote_identifier(NameStr(store->cstname))); + getRelationIdentity(&buffer, store->cstrelid, objname); + if (objname) + *objname = lappend(*objname, pstrdup(NameStr(store->cstname))); + + heap_close(cstDesc, AccessShareLock); + break; + } + case OCLASS_POLICY: { Relation polDesc; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 536c805..2315234 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -125,6 +125,7 @@ CREATE VIEW pg_tables AS pg_get_userbyid(C.relowner) AS tableowner, T.spcname AS tablespace, C.relhasindex AS hasindexes, + C.relhascstore AS hascstores, C.relhasrules AS hasrules, C.relhastriggers AS hastriggers, C.relrowsecurity AS rowsecurity @@ -139,6 +140,7 @@ CREATE VIEW pg_matviews AS pg_get_userbyid(C.relowner) AS matviewowner, T.spcname AS tablespace, C.relhasindex AS hasindexes, + C.relhascstore AS hascstores, C.relispopulated AS ispopulated, pg_get_viewdef(C.oid) AS definition FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c index 3652d7b..33ba582 100644 --- a/src/backend/catalog/toasting.c +++ b/src/backend/catalog/toasting.c @@ -279,6 +279,7 @@ create_toast_table(Relation rel, Oid toastOid, Oid toastIndexOid, rel->rd_rel->relowner, tupdesc, NIL, + NIL, RELKIND_TOASTVALUE, rel->rd_rel->relpersistence, shared_relation, diff --git a/src/backend/colstore/Makefile b/src/backend/colstore/Makefile new file mode 100644 index 0000000..d653c4e --- /dev/null +++ b/src/backend/colstore/Makefile @@ -0,0 +1,17 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for colstore +# +# IDENTIFICATION +# src/backend/colstore/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/colstore +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = vertical.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/colstore/README b/src/backend/colstore/README new file mode 100644 index 0000000..4ab5598 --- /dev/null +++ b/src/backend/colstore/README @@ -0,0 +1,187 @@ +Column Store API +================ + +The goal of the column store implementations is to allow vertical partitioning +of tables, with the benefits of + +* Increasing storage capacity thanks to better compression ratios, which is + possible because the vertical partitioning makes the data more homogenous. + This is true both for explicit compression (implemented in the column store) + or implicit (implemented at the filesystem level, thus transparent to the + database). + +* Lower I/O usage, thanks to only reading the 'vertical partitions' necessary + for the particular query, and also thanks to compression. + +* Improved CPU efficiency thanks to specialized encoding schemes. + +* Storage formats optimized for various kind of specialized devices (GPU, FPGA). + +The aim of the CS API is not to implement a column store with all those benefits +(because some of those may be actually conflicting), but providing an API that +makes it easier to implement a custom columnar storage. + +This endorses the extensibility principle, which is one of the core ideas in +PostgreSQL project. + +We do envision the API to be eventually used both internally (for built-in +column store implementations), and from extensions (with all the limitations +inherent to code delivered as an extension). + + +CREATE TABLE syntax +------------------- + +A simple CREATE TABLE statement with column store(s) might look like this: + + CREATE TABLE lineitem ( + l_orderkey BIGINT, + l_partkey INTEGER, + l_suppkey INTEGER, + l_linenumber INTEGER, + l_quantity REAL, + l_extendedprice REAL, + l_discount REAL, + l_tax REAL, + l_returnflag CHAR(1), + l_linestatus CHAR(1), + l_shipdate DATE, + l_commitdate DATE, + l_receiptdate DATE, + l_shipinstruct CHAR(25) COLUMN STORE shipinstr USING cstam1 WITH (...), + l_shipmode CHAR(10) COLUMN STORE shipmode USING cstam2 WITH (...), + l_comment VARCHAR(44) COLUMN STORE comment USING cstam2 WITH (...), + + COLUMN STORE prices + USING cstam1 (l_quantity, l_extendedprice, l_discount, l_tax), + + COLUMN STORE dates + USING cstam1 (l_shipdate, l_commitdate, l_receiptdate) + WITH (compression lzo) + +); + +If you're familiar with TPC-H benchmark, this table should be familiar to you, +this is the largest table in that data set. But take this example only as an +illustration of the syntax, not as a recommendation of how to define the column +stores in practice. + +The example defines a number of column stores - some at column level, some at +table level. The column stores defined at a column level only contain a single +column, the stores defined at table level may contain multiple columns. This is +quite similar to CHECK constraints, for example. + +The COLUMN STORE syntax for stores defined at the column level is this: + + COLUMN STORE USING WITH () + +and for stores defined at table level: + + COLUMN STORE USING () WITH () + +The is a column store name, unique within a table - once again, this is +similar to constraints (constraint names are unique in a table). + +The stands for 'access method' and references a particular implementation +of a column store API, listed in pg_cstore_am. + +Of course, is a list of columns in the column store. + +This syntax is consistent with indexes, which use the same syntax + + USING () + +Currently we only allow each column to be assigned to a single column store, +although this may be changed in the future (allowing overlapping column stores). +The columns that are not assigned to a column store remain in the heap. + + +Inheritance +----------- +TODO + + +Columns Store Handler +--------------------- + +To implement a column store, you need a implement an API defined in + + colstore/colstoreapi.h + +The design mostly follows the technical ideas from foreign data wrapper API, +so the API has a form of handler function, returning a pointer to a struct: + + typedef struct ColumnStoreRoutine + { + NodeTag type; + + /* insert a single row into the column store */ + ExecColumnStoreInsert_function ExecColumnStoreInsert; + + /* insert a batch of rows into the column store */ + ExecColumnStoreBatchInsert_function ExecColumnStoreBatchInsert; + + /* fetch values for a single row */ + ExecColumnStoreFetch_function ExecColumnStoreFetch; + + /* fetch a batch of values for a single row */ + ExecColumnStoreBatchFetch_function ExecColumnStoreBatchFetch; + + /* discard a batch of deleted rows from the column store */ + ExecColumnStoreDiscard_function ExecColumnStoreDiscard; + + /* prune the store - keep only the valid rows */ + ExecColumnStorePrune_function ExecColumnStorePrune; + + } ColumnStoreRoutine; + +that implement various tasks for querying, modification and maintenance. + +You also need to define a 'handler' which is a function that creates and +populates the routine structure with pointers to your implementation. This +function is very simple - takes no arguments and returns a pointer to the +structure as cstore_handler. + +So if the function is called my_colstore_handler(), you may do this: + + CREATE FUNCTION my_colstore_handler() + RETURNS cstore_handler + AS 'MODULE_PATHNAME' + LANGUAGE C STRICT; + +to define the handler. + + +Column Store Access Methods +--------------------------- + +The column store access method binds a name to a handler (again, this is similar +to how foreign data wrappers binds name and FDW handler). To define a new access +method, use + + CREATE COLUMN STORE ACCESS METHOD METHOD + +so using the previously defined handler, you may do + + CREATE COLUMN STORE ACCESS METHOD my_colstore METHOD my_colstore_handler + +and then use 'my_colstore' in CREATE TABLE statements. + + +Catalogs +-------- +- pg_cstore_am - access method (name + handler) +- pg_cstore - column stores defined for a table (similar to pg_index) +- pg_class - column stores defined for a table (relations) +- pg_attribute - attributes for a column store + + +Plan nodes +---------- +TODO - explain what plan nodes are supported, etc. + + +Limitations +----------- +- all column stores have to be defined at CREATE TABLE time +- each column may belong to heap or to a single column store diff --git a/src/backend/colstore/vertical.c b/src/backend/colstore/vertical.c new file mode 100644 index 0000000..4532557 --- /dev/null +++ b/src/backend/colstore/vertical.c @@ -0,0 +1,253 @@ +/*------------------------------------------------------------------------ + * vertical.c + * Simple column store implementation for POSTGRES + * + * Copyright (c) 2015, PostgreSQL Global Development Group + * + * src/backend/colstore/vertical.c + * + *------------------------------------------------------------------------ + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "catalog/heap.h" +#include "colstore/colstoreapi.h" +#include "commands/vacuum.h" +#include "storage/lmgr.h" +#include "utils/builtins.h" +#include "utils/rel.h" + +/* XXX figure out about scanDesc */ +#include "access/relscan.h" + + +typedef struct VerticalState +{ + uint32 magic; + HeapScanDesc scanDesc; + BulkInsertState bistate; +} VerticalState; + +#define VerticalColstoreMagic 0xC522C8AA + +static void vertical_open(ColumnStoreHandler *handler, + bool for_write, Snapshot snapshot); +static void vertical_insert(ColumnStoreHandler *handler, Datum *values, + bool *nulls, CommandId cid); +static void vertical_batch_insert(ColumnStoreHandler *handler, int nrows, + Datum **values, bool **nulls, CommandId cid); +static TupleTableSlot *vertical_scannext(ColumnStoreHandler *handler, + TupleTableSlot *slot); +static void vertical_endscan(ColumnStoreHandler *handler); +static void vertical_rescan(ColumnStoreHandler *handler); +static void vertical_markpos(ColumnStoreHandler *handler); +static void vertical_restrpos(ColumnStoreHandler *handler); +static void vertical_close(ColumnStoreHandler *handler); +static void vertical_truncate(Relation rel); +static int vertical_sample(Relation onerel, int elevel, HeapTuple *rows, + int targrows, double *totalrows, + double *totaldeadrows); + +Datum +vertical_cstore_handler(PG_FUNCTION_ARGS) +{ + ColumnStoreRoutine *routine = makeNode(ColumnStoreRoutine); + + routine->ExecColumnStoreOpen = vertical_open; + routine->ExecColumnStoreInsert = vertical_insert; + routine->ExecColumnStoreBatchInsert = vertical_batch_insert; + routine->ExecColumnStoreScanNext = vertical_scannext; + routine->ExecColumnStoreEndScan = vertical_endscan; + routine->ExecColumnStoreRescan = vertical_rescan; + routine->ExecColumnStoreMarkPos = vertical_markpos; + routine->ExecColumnStoreRestrPos = vertical_restrpos; + routine->ExecColumnStoreClose = vertical_close; + routine->ExecColumnStoreTruncate = vertical_truncate; + routine->ExecColumnStoreSample = vertical_sample; + + PG_RETURN_POINTER(routine); +} + +/* + * Fill in the ColumnStoreHandler opaque pointer, either in read-only mode or + * write-only, so that other routines can be invoked. + * + * Note that the colstore can opened for either read or write; a single handler + * can not do both. XXX this seems a pointless limitation. + * + * If opened for read, caller must supply a snapshot. + */ +static void +vertical_open(ColumnStoreHandler *handler, bool for_write, Snapshot snapshot) +{ + VerticalState *state = (VerticalState *) palloc0(sizeof(VerticalState)); + + state->magic = VerticalColstoreMagic; + if (for_write) + state->bistate = GetBulkInsertState(); + else + state->scanDesc = + heap_beginscan(handler->csh_relationDesc, snapshot, 0, NULL); + + handler->csh_opaque = (void *) state; +} + +/* + * Insert values for a single tuple, marked with the given command ID. + * + * XXX Note that the ItemPointer must be the first element in the values/isnull + * arrays. This is an ugly crock which must be removed. + */ +static void +vertical_insert(ColumnStoreHandler *handler, Datum *values, bool *isnull, + CommandId cid) +{ + VerticalState *state = (VerticalState *) handler->csh_opaque; + HeapTuple tuple; + TupleDesc tupdesc; + + if (state->magic != VerticalColstoreMagic) + ereport(ERROR, + (errmsg("unsightly pointer"))); + Assert(state->bistate); + + tupdesc = RelationGetDescr(handler->csh_relationDesc); + tuple = heap_form_tuple(tupdesc, values, isnull); + + heap_insert(handler->csh_relationDesc, tuple, cid, + 0 /* XXX no options --- OK? */, + state->bistate); + heap_freetuple(tuple); +} + +/* + * Insert values for multiple tuples, as above. + */ +static void +vertical_batch_insert(ColumnStoreHandler *handler, int nrows, Datum **values, + bool **isnull, CommandId cid) +{ + VerticalState *state = (VerticalState *) handler->csh_opaque; + int i; + TupleDesc tupdesc; + + if (state->magic != VerticalColstoreMagic) + ereport(ERROR, + (errmsg("unsightly pointer"))); + Assert(state->bistate); + tupdesc = RelationGetDescr(handler->csh_relationDesc); + + for (i = 0; i < nrows; i++) + { + HeapTuple tuple; + + tuple = heap_form_tuple(tupdesc, values[i], isnull[i]); + + heap_insert(handler->csh_relationDesc, tuple, cid, + 0, state->bistate); + heap_freetuple(tuple); + } +} + +static TupleTableSlot * +vertical_scannext(ColumnStoreHandler *handler, TupleTableSlot *slot) +{ + VerticalState *state = (VerticalState *) handler->csh_opaque; + HeapTuple tuple; + + if (state->magic != VerticalColstoreMagic) + ereport(ERROR, + (errmsg("unsightly pointer"))); + Assert(state->scanDesc); + + tuple = heap_getnext(state->scanDesc, ForwardScanDirection); + if (tuple) + ExecStoreTuple(tuple, + slot, + state->scanDesc->rs_cbuf, + false); + else + ExecClearTuple(slot); + + return slot; +} + +static void +vertical_endscan(ColumnStoreHandler *handler) +{ + VerticalState *state = (VerticalState *) handler->csh_opaque; + + if (state->magic != VerticalColstoreMagic) + ereport(ERROR, + (errmsg("unsightly pointer"))); + Assert(state->scanDesc); + + /* + * close heap scan + */ + heap_endscan(state->scanDesc); +} + +static void +vertical_rescan(ColumnStoreHandler *handler) +{ + VerticalState *state = (VerticalState *) handler->csh_opaque; + HeapScanDesc scan; + + + if (state->magic != VerticalColstoreMagic) + ereport(ERROR, + (errmsg("unsightly pointer"))); + Assert(state->scanDesc); + + scan = state->scanDesc; + + heap_rescan(scan, /* scan desc */ + NULL); /* new scan keys */ +} + +static void +vertical_markpos(ColumnStoreHandler *handler) +{ + +} + +static void +vertical_restrpos(ColumnStoreHandler *handler) +{ + +} + +/* + * Release resources + */ +static void +vertical_close(ColumnStoreHandler *handler) +{ + VerticalState *state = (VerticalState *) handler->csh_opaque; + + if (state->magic != VerticalColstoreMagic) + ereport(ERROR, + (errmsg("unsightly pointer"))); + + /* Release all resources */ + if (state->scanDesc) + heap_endscan(state->scanDesc); + FreeBulkInsertState(state->bistate); + pfree(state); +} + +static void +vertical_truncate(Relation rel) +{ + heap_truncate_one_rel(rel); +} + +static int +vertical_sample(Relation onerel, int elevel, HeapTuple *rows, int targrows, + double *totalrows, double *totaldeadrows) +{ + return acquire_sample_rows(onerel, elevel, rows, targrows, totalrows, + totaldeadrows); +} diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index b1ac704..d2bb636 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -14,7 +14,7 @@ include $(top_builddir)/src/Makefile.global OBJS = aggregatecmds.o alter.o analyze.o async.o cluster.o comment.o \ collationcmds.o constraint.o conversioncmds.o copy.o createas.o \ - dbcommands.o define.o discard.o dropcmds.o \ + colstorecmds.o dbcommands.o define.o discard.o dropcmds.o \ event_trigger.o explain.o extension.o foreigncmds.o functioncmds.o \ indexcmds.o lockcmds.o matview.o operatorcmds.o opclasscmds.o \ policy.o portalcmds.o prepare.o proclang.o \ diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index ddb68ab..54d2116 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -28,6 +28,7 @@ #include "catalog/pg_collation.h" #include "catalog/pg_inherits_fn.h" #include "catalog/pg_namespace.h" +#include "colstore/colstoreapi.h" #include "commands/dbcommands.h" #include "commands/tablecmds.h" #include "commands/vacuum.h" @@ -85,15 +86,11 @@ static void compute_index_stats(Relation onerel, double totalrows, MemoryContext col_context); static VacAttrStats *examine_attribute(Relation onerel, int attnum, Node *index_expr); -static int acquire_sample_rows(Relation onerel, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, double *totaldeadrows); static int compare_rows(const void *a, const void *b); static int acquire_inherited_sample_rows(Relation onerel, int elevel, HeapTuple *rows, int targrows, double *totalrows, double *totaldeadrows); -static void update_attstats(Oid relid, bool inh, - int natts, VacAttrStats **vacattrstats); +static void update_attstats(bool inh, int natts, VacAttrStats **vacattrstats); static Datum std_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull); static Datum ind_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull); @@ -501,6 +498,7 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, { MemoryContext col_context, old_context; + bool gotoffheap = false; col_context = AllocSetContextCreate(anl_context, "Analyze Column", @@ -514,6 +512,13 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, VacAttrStats *stats = vacattrstats[i]; AttributeOpts *aopt; + /* Don't attempt to analyze off-heap attributes */ + if (stats->attr->attphynum == InvalidAttrNumber) + { + gotoffheap = true; + continue; + } + stats->rows = rows; stats->tupDesc = onerel->rd_att; (*stats->compute_stats) (stats, @@ -544,6 +549,100 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, rows, numrows, col_context); + if (gotoffheap) + { + Bitmapset *cstores = NULL; + + RelationGetColStoreList(onerel); + + if (!onerel->rd_cstlist) + elog(ERROR, "Found off heap columns on relation which has no column stores"); + + for (i = 0; i < attr_cnt; i++) + { + VacAttrStats *stats = vacattrstats[i]; + ListCell *l; + int cstoreidx; + + if (stats->attr->attphynum != InvalidAttrNumber) + continue; + + cstoreidx = 0; + foreach(l, onerel->rd_cstlist) + { + Oid cstoreoid = lfirst_oid(l); + Relation crel; + int col; + bool found = false; + + /* Have we already decided to analyze this column store? */ + if (bms_is_member(cstoreidx, cstores)) + continue; + + crel = relation_open(cstoreoid, NoLock); + for (col = 0; col < crel->rd_cstore->cstnatts; col++) + { + if (crel->rd_cstore->cstatts.values[col] == stats->attr->attnum) + { + Form_pg_attribute newattr = crel->rd_att->attrs[col + 1]; + + newattr->attstattarget = stats->attr->attstattarget; + stats->tupattnum = newattr->attnum; + stats->tupDesc = crel->rd_att; + stats->attr = newattr; + + cstores = bms_add_member(cstores, cstoreidx); + found = true; + break; + } + } + relation_close(crel, NoLock); + + if (found) + break; + } + } + + if (cstores != NULL) + { + int cstoreidx = - 1; + + while ((cstoreidx = bms_next_member(cstores, cstoreidx)) >= 0) + { + Oid cstoreoid = list_nth_oid(onerel->rd_cstlist, cstoreidx); + Relation crel; + ColumnStoreRoutine *routine; + crel = relation_open(cstoreoid, AccessShareLock); + + routine = GetColumnStoreRoutineForRelation(crel, false); + + if (routine->ExecColumnStoreSample == NULL) + elog(ERROR, "Sample routine not supplied by column store AM"); + + routine->ExecColumnStoreSample(crel, elevel, rows, + targrows, &totalrows, &totaldeadrows); + + for (i = 0; i < attr_cnt; i++) + { + VacAttrStats *stats = vacattrstats[i]; + + if (stats->attr->attrelid == cstoreoid) + { + stats->rows = rows; + (*stats->compute_stats) (stats, + std_fetch_func, + numrows, + totalrows); + } + } + + relation_close(crel, AccessShareLock); + } + } + } + + + MemoryContextSwitchTo(old_context); MemoryContextDelete(col_context); @@ -552,15 +651,13 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, * previous statistics for the target columns. (If there are stats in * pg_statistic for columns we didn't process, we leave them alone.) */ - update_attstats(RelationGetRelid(onerel), inh, - attr_cnt, vacattrstats); + update_attstats(inh, attr_cnt, vacattrstats); for (ind = 0; ind < nindexes; ind++) { AnlIndexData *thisdata = &indexdata[ind]; - update_attstats(RelationGetRelid(Irel[ind]), false, - thisdata->attr_cnt, thisdata->vacattrstats); + update_attstats(false, thisdata->attr_cnt, thisdata->vacattrstats); } } @@ -969,7 +1066,7 @@ examine_attribute(Relation onerel, int attnum, Node *index_expr) * block. The previous sampling method put too much credence in the row * density near the start of the table. */ -static int +int acquire_sample_rows(Relation onerel, int elevel, HeapTuple *rows, int targrows, double *totalrows, double *totaldeadrows) @@ -1472,7 +1569,7 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, * by taking a self-exclusive lock on the relation in analyze_rel(). */ static void -update_attstats(Oid relid, bool inh, int natts, VacAttrStats **vacattrstats) +update_attstats(bool inh, int natts, VacAttrStats **vacattrstats) { Relation sd; int attno; @@ -1507,7 +1604,7 @@ update_attstats(Oid relid, bool inh, int natts, VacAttrStats **vacattrstats) replaces[i] = true; } - values[Anum_pg_statistic_starelid - 1] = ObjectIdGetDatum(relid); + values[Anum_pg_statistic_starelid - 1] = ObjectIdGetDatum(stats->attr->attrelid); values[Anum_pg_statistic_staattnum - 1] = Int16GetDatum(stats->attr->attnum); values[Anum_pg_statistic_stainherit - 1] = BoolGetDatum(inh); values[Anum_pg_statistic_stanullfrac - 1] = Float4GetDatum(stats->stanullfrac); @@ -1571,7 +1668,7 @@ update_attstats(Oid relid, bool inh, int natts, VacAttrStats **vacattrstats) /* Is there already a pg_statistic tuple for this attribute? */ oldtup = SearchSysCache3(STATRELATTINH, - ObjectIdGetDatum(relid), + ObjectIdGetDatum(stats->attr->attrelid), Int16GetDatum(stats->attr->attnum), BoolGetDatum(inh)); diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 7ab4874..c40a575 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -25,6 +25,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -667,6 +668,7 @@ make_new_heap(Oid OIDOldHeap, Oid NewTableSpace, char relpersistence, OldHeap->rd_rel->relowner, OldHeapDesc, NIL, + CloneColumnStores(OldHeap), RELKIND_RELATION, relpersistence, false, diff --git a/src/backend/commands/colstorecmds.c b/src/backend/commands/colstorecmds.c new file mode 100644 index 0000000..eabcced --- /dev/null +++ b/src/backend/commands/colstorecmds.c @@ -0,0 +1,177 @@ +/*------------------------------------------------------------------------- + * + * colstorecmds.c + * column store creation/manipulation commands + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/commands/colstorecmds.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/reloptions.h" +#include "access/xact.h" +#include "catalog/colstore.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_cstore_am.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "colstore/colstoreapi.h" +#include "miscadmin.h" +#include "parser/parse_func.h" +#include "tcop/utility.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + + +/* + * Convert a handler function name passed from the parser to an Oid. + */ +static Oid +lookup_cstore_handler_func(DefElem *handler) +{ + Oid handlerOid; + Oid funcargtypes[1]; + + if (handler == NULL || handler->arg == NULL) + return InvalidOid; + + /* handlers have no arguments */ + handlerOid = LookupFuncName((List *) handler->arg, 0, funcargtypes, false); + + /* check that handler has correct return type */ + if (get_func_rettype(handlerOid) != CSTORE_HANDLEROID) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("function %s must return type \"cstore_handler\"", + NameListToString((List *) handler->arg)))); + + return handlerOid; +} + +/* + * Process function options of CREATE COLUMN STORE ACCESS METHOD + */ +static void +parse_func_options(List *func_options, + bool *handler_given, Oid *csthandler) +{ + ListCell *cell; + + *handler_given = false; + *csthandler = InvalidOid; + + foreach(cell, func_options) + { + DefElem *def = (DefElem *) lfirst(cell); + + if (strcmp(def->defname, "handler") == 0) + { + if (*handler_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + *handler_given = true; + *csthandler = lookup_cstore_handler_func(def); + } + else + elog(ERROR, "option \"%s\" not recognized", + def->defname); + } +} + +/* + * Create a column store access method + */ +ObjectAddress +CreateColumnStoreAM(CreateColumnStoreAMStmt *stmt) +{ + Relation rel; + Datum values[Natts_pg_cstore_am]; + bool nulls[Natts_pg_cstore_am]; + HeapTuple tuple; + Oid cstamId; + bool handler_given; + Oid csthandler; + ObjectAddress myself; + ObjectAddress referenced; + + rel = heap_open(CStoreAmRelationId, RowExclusiveLock); + + /* Must be super user */ + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to create column store access method \"%s\"", + stmt->cstamname), + errhint("Must be superuser to create a column store access method."))); + + /* + * Check that there is no other column store AM by this name. + */ + if (GetColumnStoreAMByName(stmt->cstamname, true) != InvalidOid) + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("column store access method \"%s\" already exists", + stmt->cstamname))); + + /* + * Insert tuple into pg_cstore_am. + */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + values[Anum_pg_cstore_am_cstamname - 1] = + DirectFunctionCall1(namein, CStringGetDatum(stmt->cstamname)); + + /* Lookup handler and validator functions, if given */ + parse_func_options(stmt->func_options, &handler_given, &csthandler); + + /* XXX ereport */ + if (!handler_given) + elog(ERROR, "column store access method requires HANDLER option"); + + values[Anum_pg_cstore_am_cstamhandler - 1] = ObjectIdGetDatum(csthandler); + + tuple = heap_form_tuple(rel->rd_att, values, nulls); + + cstamId = simple_heap_insert(rel, tuple); + CatalogUpdateIndexes(rel, tuple); + + heap_freetuple(tuple); + + /* record dependencies */ + myself.classId = CStoreAmRelationId; + myself.objectId = cstamId; + myself.objectSubId = 0; + + if (OidIsValid(csthandler)) + { + referenced.classId = ProcedureRelationId; + referenced.objectId = csthandler; + referenced.objectSubId = 0; + recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + } + + /* dependency on extension */ + recordDependencyOnCurrentExtension(&myself, false); + + /* Post creation hook for new column store access method */ + InvokeObjectPostCreateHook(CStoreAmRelationId, cstamId, 0); + + heap_close(rel, RowExclusiveLock); + + return myself; +} diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 7dbe04e..931f85f 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -25,6 +25,7 @@ #include "access/sysattr.h" #include "access/xact.h" #include "access/xlog.h" +#include "catalog/colstore.h" #include "catalog/pg_type.h" #include "commands/copy.h" #include "commands/defrem.h" @@ -298,7 +299,7 @@ static void CopyFromInsertBatch(CopyState cstate, EState *estate, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, - int firstBufferedLineNo); + int firstBufferedLineNo, TupleDesc tupDesc); static bool CopyReadLine(CopyState cstate); static bool CopyReadLineText(CopyState cstate); static int CopyReadAttributesText(CopyState cstate); @@ -834,6 +835,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed) rte->rtekind = RTE_RELATION; rte->relid = RelationGetRelid(rel); rte->relkind = rel->rd_rel->relkind; + rte->relhascstore = rel->rd_rel->relhascstore; rte->requiredPerms = required_access; range_table = list_make1(rte); @@ -2342,6 +2344,8 @@ CopyFrom(CopyState cstate) ExecOpenIndices(resultRelInfo, false); + ExecOpenColumnStores(resultRelInfo); + estate->es_result_relations = resultRelInfo; estate->es_num_result_relations = 1; estate->es_result_relation_info = resultRelInfo; @@ -2423,7 +2427,24 @@ CopyFrom(CopyState cstate) break; /* And now we can form the input tuple. */ - tuple = heap_form_tuple(tupDesc, values, nulls); + switch (tupDesc->tdattorder) + { + /* + * If physical matches logical then we'll just form a logical tuple + * as the function does not have to perform any translation between + * logical and physical orders. Likewise if there are any off-heap + * attributes, as in this case we can't form a physical tuple as we + * would end up throwing away the off-heap attributes. + */ + case ATTRORDER_PHYSMATCHLOGICAL: + case ATTRORDER_OFFHEAPATTRS: + tuple = heap_form_logical_tuple(tupDesc, values, nulls); + break; + + /* Contains out of outer attributes */ + default: + tuple = heap_form_tuple(tupDesc, values, nulls); + } if (loaded_oid != InvalidOid) HeapTupleSetOid(tuple, loaded_oid); @@ -2481,7 +2502,7 @@ CopyFrom(CopyState cstate) CopyFromInsertBatch(cstate, estate, mycid, hi_options, resultRelInfo, myslot, bistate, nBufferedTuples, bufferedTuples, - firstBufferedLineNo); + firstBufferedLineNo, tupDesc); nBufferedTuples = 0; bufferedTuplesSize = 0; } @@ -2490,6 +2511,12 @@ CopyFrom(CopyState cstate) { List *recheckIndexes = NIL; + /* + * FIXME This needs to handle the column stores (it's not + * handled by the batching code because of the before + * row insert triggers or something). + */ + /* OK, store the tuple and create index entries for it */ heap_insert(cstate->rel, tuple, mycid, hi_options, bistate); @@ -2519,7 +2546,7 @@ CopyFrom(CopyState cstate) CopyFromInsertBatch(cstate, estate, mycid, hi_options, resultRelInfo, myslot, bistate, nBufferedTuples, bufferedTuples, - firstBufferedLineNo); + firstBufferedLineNo, tupDesc); /* Done, clean up */ error_context_stack = errcallback.previous; @@ -2548,6 +2575,8 @@ CopyFrom(CopyState cstate) ExecCloseIndices(resultRelInfo); + ExecCloseColumnStores(resultRelInfo); + FreeExecutorState(estate); /* @@ -2570,7 +2599,7 @@ CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, int hi_options, ResultRelInfo *resultRelInfo, TupleTableSlot *myslot, BulkInsertState bistate, int nBufferedTuples, HeapTuple *bufferedTuples, - int firstBufferedLineNo) + int firstBufferedLineNo, TupleDesc tupDesc) { MemoryContext oldcontext; int i; @@ -2588,14 +2617,80 @@ CopyFromInsertBatch(CopyState cstate, EState *estate, CommandId mycid, * before calling it. */ oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); - heap_multi_insert(cstate->rel, - bufferedTuples, - nBufferedTuples, - mycid, - hi_options, - bistate); + + /* + * If the physical attribute order matches the logcal, then we can simply + * insert the tuples. If the attributes in the tuple are not in logical + * order then we'll have built the tuple in physical order in the calling + * function anyway, so we can also just insert the tuples. + */ + if (tupDesc->tdattorder == ATTRORDER_PHYSMATCHLOGICAL || + tupDesc->tdattorder == ATTRORDER_OUTOFORDER) + { + heap_multi_insert(cstate->rel, + bufferedTuples, + nBufferedTuples, + mycid, + hi_options, + bistate); + } + else + { + HeapTuple *diskTuples; + Datum *values; + bool *isnull; + + int nattrs = tupDesc->natts; + + /* + * Tuples require translation from the logical tuple order, into + * on-disk physical tuple order. Perhaps there is a better way to do + * this, maybe we could delay forming the tuple and only ever form a + * physical tuple? + */ + diskTuples = (HeapTuple *) palloc(nBufferedTuples * sizeof(HeapTuple)); + values = (Datum *) palloc(nattrs * sizeof(Datum)); + isnull = (bool *) palloc(nattrs * sizeof(bool)); + + for (i = 0; i < nBufferedTuples; i++) + { + int attr; + + for (attr = 0; attr < nattrs; attr++) + values[attr] = heap_getlogattr(bufferedTuples[i], attr + 1, tupDesc, &isnull[attr]); + + diskTuples[i] = heap_form_tuple(tupDesc, values, isnull); + } + + /* + * Clear any cached attribute offsets from reading the logical tuple, + * as these will be the wrong value for reading the physical tuple. + */ + TupleDescCacheReset(tupDesc); + + heap_multi_insert(cstate->rel, + diskTuples, + nBufferedTuples, + mycid, + hi_options, + bistate); + + /* + * Harvest the TIDs from the inserted tuples, we need to apply these + * to the column store in order to link the records + */ + for(i = 0; i < nBufferedTuples; i++) + ItemPointerCopy(&diskTuples[i]->t_self, &bufferedTuples[i]->t_self); + + if (resultRelInfo->ri_ColumnStoreHandler == NIL) + elog(ERROR, "TupleDesc has off-heap attibutes but relation has no column stores"); + + ExecBatchInsertColStoreTuples(nBufferedTuples, bufferedTuples, estate); + + } MemoryContextSwitchTo(oldcontext); + /* * If there are any indexes, update them for all the inserted tuples, and * run AFTER ROW INSERT triggers. diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 41183f6..a139042 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -425,11 +425,15 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo) * * XXX: It would arguably make sense to skip this check if into->skipData * is true. + * + * Note: it's okay for relhascstore to be inaccurate, since it's only used + * for permissions checking. */ rte = makeNode(RangeTblEntry); rte->rtekind = RTE_RELATION; rte->relid = intoRelationAddr.objectId; rte->relkind = relkind; + rte->relhascstore = false; rte->requiredPerms = ACL_INSERT; for (attnum = 1; attnum <= intoRelationDesc->rd_att->natts; attnum++) diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index 3d1cb0b..0488ea9 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -90,6 +90,7 @@ static event_trigger_support_data event_trigger_support[] = { {"CAST", true}, {"CONSTRAINT", true}, {"COLLATION", true}, + {"COLUMN STORE ACCESS METHOD", true}, {"CONVERSION", true}, {"DATABASE", false}, {"DOMAIN", true}, @@ -1118,6 +1119,8 @@ EventTriggerSupportsObjectType(ObjectType obtype) case OBJECT_USER_MAPPING: case OBJECT_VIEW: return true; + case OBJECT_COLSTORE: + elog(ERROR, "unsupported --- XXX fill this in"); } return true; } @@ -1168,6 +1171,8 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_EXTENSION: case OCLASS_POLICY: return true; + case OCLASS_COLSTORE: + elog(ERROR, "unsupported --- XXX fill this in"); } return true; diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 12dae77..97a7fd6 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -898,6 +898,9 @@ ExplainNode(PlanState *planstate, List *ancestors, else pname = sname; break; + case T_ColumnStoreScan: + pname = sname = "Column Store Scan"; + break; case T_Material: pname = sname = "Materialize"; break; @@ -1018,6 +1021,7 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_ValuesScan: case T_CteScan: case T_WorkTableScan: + case T_ColumnStoreScan: ExplainScanTarget((Scan *) plan, es); break; case T_ForeignScan: @@ -1279,6 +1283,7 @@ ExplainNode(PlanState *planstate, List *ancestors, case T_CteScan: case T_WorkTableScan: case T_SubqueryScan: + case T_ColumnStoreScan: show_scan_qual(plan->qual, "Filter", planstate, ancestors, es); if (plan->qual) show_instrumentation_count("Rows Removed by Filter", 1, diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index a217dbc..86bf5ee 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -23,6 +23,7 @@ #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "catalog/colstore.h" #include "catalog/dependency.h" #include "catalog/heap.h" #include "catalog/index.h" @@ -270,7 +271,8 @@ struct DropRelationCallbackState static void truncate_check_rel(Relation rel); static List *MergeAttributes(List *schema, List *supers, char relpersistence, - List **supOids, List **supconstr, int *supOidCount); + List **supOids, List **supconstr, int *supOidCount, + List **colstores); static bool MergeCheckConstraint(List *constraints, char *name, Node *expr); static void MergeAttributesIntoExisting(Relation child_rel, Relation parent_rel); static void MergeConstraintsIntoExisting(Relation child_rel, Relation parent_rel); @@ -462,6 +464,9 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, Oid tablespaceId; Relation rel; TupleDesc descriptor; + List *decl_cstores = NIL, + *inh_cstores = NIL, + *colstores; List *inheritOids; List *old_constraints; bool localHasOids; @@ -571,19 +576,43 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, ofTypeId = InvalidOid; /* + * Initialize the list of column stores with the ones provided in + * table constraint form. + */ + foreach(listptr, stmt->colstores) + { + ColumnStoreClause *clause = (ColumnStoreClause *) lfirst(listptr); + ColumnStoreClauseInfo *store = palloc(sizeof(ColumnStoreClauseInfo)); + + store->cstoreClause = clause; + store->attnum = InvalidAttrNumber; + store->attnums = NIL; + store->cstoreOid = InvalidOid; + + decl_cstores = lappend(decl_cstores, store); + } + + /* * Look up inheritance ancestors and generate relation schema, including - * inherited attributes. + * inherited attributes. Add column stores coming from parent rels. */ schema = MergeAttributes(schema, stmt->inhRelations, stmt->relation->relpersistence, - &inheritOids, &old_constraints, &parentOidCount); + &inheritOids, &old_constraints, &parentOidCount, + &inh_cstores); /* * Create a tuple descriptor from the relation schema. Note that this * deals with column names, types, and NOT NULL constraints, but not * default values or CHECK constraints; we handle those below. */ - descriptor = BuildDescForRelation(schema); + descriptor = BuildDescForRelation(schema, &decl_cstores); + + /* + * Determine the column stores we need. + */ + colstores = DetermineColumnStores(descriptor, decl_cstores, inh_cstores, + tablespaceId); /* * Notice that we allow OIDs here only for plain tables, even though some @@ -665,6 +694,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, descriptor, list_concat(cookedDefaults, old_constraints), + colstores, relkind, stmt->relation->relpersistence, false, @@ -1238,9 +1268,12 @@ ExecuteTruncate(TruncateStmt *stmt) } /* - * Reconstruct the indexes to match, and we're done. + * Reconstruct the indexes to match. */ reindex_relation(heap_relid, REINDEX_REL_PROCESS_TOAST, 0); + + if (rel->rd_rel->relhascstore) + ExecTruncateColumnStores(rel); } pgstat_count_truncate(rel); @@ -1363,6 +1396,7 @@ storage_name(char c) * 'supconstr' receives a list of constraints belonging to the parents, * updated as necessary to be valid for the child. * 'supOidCount' is set to the number of parents that have OID columns. + * 'colstores' is appended ColumnStoreClauseInfo structs from parent rels. * * Return value: * Completed schema list. @@ -1408,7 +1442,8 @@ storage_name(char c) */ static List * MergeAttributes(List *schema, List *supers, char relpersistence, - List **supOids, List **supconstr, int *supOidCount) + List **supOids, List **supconstr, int *supOidCount, + List **colstores) { ListCell *entry; List *inhSchema = NIL; @@ -1506,6 +1541,8 @@ MergeAttributes(List *schema, List *supers, char relpersistence, TupleConstr *constr; AttrNumber *newattno; AttrNumber parent_attno; + List *pstores; + ListCell *cell; /* * A self-exclusive lock is needed here. If two backends attempt to @@ -1664,6 +1701,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence, def->collClause = NULL; def->collOid = attribute->attcollation; def->constraints = NIL; + def->cstoreClause = NULL; def->location = -1; inhSchema = lappend(inhSchema, def); newattno[parent_attno - 1] = ++child_attno; @@ -1772,6 +1810,38 @@ MergeAttributes(List *schema, List *supers, char relpersistence, } } + /* + * Process column stores in the parent, using the completed + * newattno map. + */ + pstores = RelationGetColStoreList(relation); + foreach(cell, pstores) + { + Oid cstoreid = lfirst_oid(cell); + Relation storerel; + Form_pg_cstore cst; + ColumnStoreClauseInfo *cstinfo; + int i; + + /* AccessShare should be sufficient, since we hold lock on rel */ + storerel = relation_open(cstoreid, AccessShareLock); + cst = storerel->rd_cstore; + + cstinfo = palloc(sizeof(ColumnStoreClauseInfo)); + cstinfo->attnum = InvalidAttrNumber; + cstinfo->cstoreClause = NULL; + cstinfo->cstoreOid = RelationGetRelid(storerel); + cstinfo->attnums = NIL; + for (i = 0; i < cst->cstnatts; i++) + cstinfo->attnums = lappend_int(cstinfo->attnums, + newattno[cst->cstatts.values[i]-1]); + + relation_close(storerel, AccessShareLock); + + *colstores = lappend(*colstores, cstinfo); + } + list_free(pstores); + pfree(newattno); /* @@ -1861,6 +1931,8 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(newdef->storage)))); + /* FIXME see about merging cstore decl here */ + /* Mark the column as locally defined */ def->is_local = true; /* Merge of NOT NULL constraints = OR 'em together */ @@ -4842,6 +4914,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, attribute.attcacheoff = -1; attribute.atttypmod = typmod; attribute.attnum = newattnum; + attribute.attphynum = newattnum; attribute.attbyval = tform->typbyval; attribute.attndims = list_length(colDef->typeName->arrayBounds); attribute.attstorage = tform->typstorage; diff --git a/src/backend/commands/trigger.c b/src/backend/commands/trigger.c index 43421d6..d9cb227 100644 --- a/src/backend/commands/trigger.c +++ b/src/backend/commands/trigger.c @@ -3884,6 +3884,7 @@ afterTriggerInvokeEvents(AfterTriggerEventList *events, /* Close indices and then the relation itself */ ExecCloseIndices(resultRelInfo); + ExecCloseColumnStores(resultRelInfo); heap_close(resultRelInfo->ri_RelationDesc, NoLock); } FreeExecutorState(estate); diff --git a/src/backend/commands/view.c b/src/backend/commands/view.c index efa4be1..d7b702b 100644 --- a/src/backend/commands/view.c +++ b/src/backend/commands/view.c @@ -170,7 +170,7 @@ DefineVirtualRelation(RangeVar *relation, List *tlist, bool replace, * verify that the old column list is an initial prefix of the new * column list. */ - descriptor = BuildDescForRelation(attrList); + descriptor = BuildDescForRelation(attrList, NULL); checkViewTupleDesc(descriptor, rel->rd_att); /* diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile index 51edd4c..51f9e1a 100644 --- a/src/backend/executor/Makefile +++ b/src/backend/executor/Makefile @@ -18,6 +18,7 @@ OBJS = execAmi.o execCurrent.o execGrouping.o execIndexing.o execJunk.o \ execUtils.o functions.o instrument.o nodeAppend.o nodeAgg.o \ nodeBitmapAnd.o nodeBitmapOr.o \ nodeBitmapHeapscan.o nodeBitmapIndexscan.o nodeCustom.o nodeGather.o \ + execColumnStore.o nodeColumnStorescan.o \ nodeHash.o nodeHashjoin.o nodeIndexscan.o nodeIndexonlyscan.o \ nodeLimit.o nodeLockRows.o \ nodeMaterial.o nodeMergeAppend.o nodeMergejoin.o nodeModifyTable.o \ diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c index b969fc0..df4209a 100644 --- a/src/backend/executor/execAmi.c +++ b/src/backend/executor/execAmi.c @@ -20,6 +20,7 @@ #include "executor/nodeBitmapHeapscan.h" #include "executor/nodeBitmapIndexscan.h" #include "executor/nodeBitmapOr.h" +#include "executor/nodeColumnStorescan.h" #include "executor/nodeCtescan.h" #include "executor/nodeCustom.h" #include "executor/nodeForeignscan.h" @@ -213,6 +214,10 @@ ExecReScan(PlanState *node) ExecReScanCustomScan((CustomScanState *) node); break; + case T_ColumnStoreScanState: + ExecReScanColumnStoreScan((ColumnStoreScanState *) node); + break; + case T_NestLoopState: ExecReScanNestLoop((NestLoopState *) node); break; @@ -307,6 +312,10 @@ ExecMarkPos(PlanState *node) ExecCustomMarkPos((CustomScanState *) node); break; + case T_ColumnStoreScanState: + ExecColumnStoreScanMarkPos((ColumnStoreScanState *) node); + break; + case T_MaterialState: ExecMaterialMarkPos((MaterialState *) node); break; @@ -356,6 +365,10 @@ ExecRestrPos(PlanState *node) ExecCustomRestrPos((CustomScanState *) node); break; + case T_ColumnStoreScanState: + ExecColumnStoreScanRestrPos((ColumnStoreScanState *) node); + break; + case T_MaterialState: ExecMaterialRestrPos((MaterialState *) node); break; @@ -397,6 +410,10 @@ ExecSupportsMarkRestore(Path *pathnode) case T_Sort: return true; + case T_ColumnStoreScan: + /* XXX perhaps check the API methods are not NULL here? */ + return true; + case T_CustomScan: Assert(IsA(pathnode, CustomPath)); if (((CustomPath *) pathnode)->flags & CUSTOMPATH_SUPPORT_MARK_RESTORE) @@ -506,6 +523,7 @@ ExecSupportsBackwardScan(Plan *node) } return false; + case T_ColumnStoreScan: case T_Material: case T_Sort: /* these don't evaluate tlist */ diff --git a/src/backend/executor/execColumnStore.c b/src/backend/executor/execColumnStore.c new file mode 100644 index 0000000..01cd721 --- /dev/null +++ b/src/backend/executor/execColumnStore.c @@ -0,0 +1,274 @@ +/*------------------------------------------------------------------------- + * + * execColumnStore.c + * routines for inserting tuples into column stores. + * + * ExecInsertColStoreTuples() is the main entry point. It's called after + * inserting a tuple to the heap, and it inserts corresponding values + * into all column stores. + * + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/executor/execColumnStore.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/relscan.h" +#include "catalog/colstore.h" +#include "colstore/colstoreapi.h" +#include "executor/executor.h" +#include "nodes/nodeFuncs.h" +#include "storage/lmgr.h" +#include "utils/memutils.h" +#include "utils/tqual.h" + + +/* ---------------------------------------------------------------- + * ExecOpenColumnStores + * + * Find the column stores associated with a result relation, open them, + * and save information about them in the result ResultRelInfo. + * + * At entry, caller has already opened and locked + * resultRelInfo->ri_RelationDesc. + * ---------------------------------------------------------------- + */ +void +ExecOpenColumnStores(ResultRelInfo *resultRelInfo) +{ + Relation resultRelation = resultRelInfo->ri_RelationDesc; + List *colstoreoidlist; + ListCell *l; + + /* fast path if no column stores */ + if (!RelationGetForm(resultRelation)->relhascstore) + { + resultRelInfo->ri_ColumnStoreHandler = NIL; + return; + } + + /* + * Get cached list of colstore OIDs; bail out if there are no colstores + * after all. + */ + colstoreoidlist = RelationGetColStoreList(resultRelation); + if (colstoreoidlist == NIL) + { + resultRelInfo->ri_ColumnStoreHandler = NIL; + return; + } + + /* Open each colstore and stash it into the list */ + foreach(l, colstoreoidlist) + { + Oid cstoreOid = lfirst_oid(l); + Relation cstoreDesc; + ColumnStoreHandler *handler; + LOCKMODE lockmode = RowExclusiveLock; + + cstoreDesc = relation_open(cstoreOid, lockmode); + + /* Build the column store handler */ + handler = BuildColumnStoreHandler(cstoreDesc, true, lockmode, NULL); + + /* add it to the list */ + resultRelInfo->ri_ColumnStoreHandler = + lappend(resultRelInfo->ri_ColumnStoreHandler, handler); + } + + list_free(colstoreoidlist); +} + +/* ---------------------------------------------------------------- + * ExecCloseColumnStores + * + * Close the column store relations stored in resultRelInfo + * ---------------------------------------------------------------- + */ +void +ExecCloseColumnStores(ResultRelInfo *resultRelInfo) +{ + ListCell *l; + + foreach (l, resultRelInfo->ri_ColumnStoreHandler) + { + ColumnStoreHandler *handler = lfirst(l); + + CloseColumnStore(handler); + } + /* + * resetting the list to NIL avoids keeping dangling pointers around, but + * risks confusing code into thinking this relation has no column stores. + * Therefore we set the list to an invalid pointer instead. + */ + resultRelInfo->ri_ColumnStoreHandler = (List *) 0x7f; +} + +/* ---------------------------------------------------------------- + * ExecInsertColStoreTuples + * + * This routine takes care of inserting column store tuples + * into all the relations vertically partitioning the result relation + * when a heap tuple is inserted into the result relation. + * + * CAUTION: this must not be called for a HOT update. + * We can't defend against that here for lack of info. + * Should we change the API to make it safer? + * ---------------------------------------------------------------- + */ +void +ExecInsertColStoreTuples(HeapTuple tuple, EState *estate) +{ + ResultRelInfo *resultRelInfo; + Datum values[INDEX_MAX_KEYS]; /* FIXME INDEX_MAX_KEYS=32 seems a bit low */ + bool isnull[INDEX_MAX_KEYS]; + TupleDesc tupdesc; + ListCell *l; + + /* + * Get information from the result relation info structure. + */ + resultRelInfo = estate->es_result_relation_info; + tupdesc = RelationGetDescr(resultRelInfo->ri_RelationDesc); + + foreach(l, resultRelInfo->ri_ColumnStoreHandler) + { + ColumnStoreHandler *handler = lfirst(l); + ExecColumnStoreInsert_function insert; + + insert = handler->csh_ColumnStoreRoutine->ExecColumnStoreInsert; + if (insert == NULL) + elog(ERROR, "Insert routine not supplied by column store AM"); + + /* Obtain the data arrays for the column store */ + FormColumnStoreDatum(handler, tuple, tupdesc, values, isnull); + + /* And insert them */ + insert(handler, values, isnull, estate->es_output_cid); + } +} + +/* same thing, but for N tuples. XXX comment some more */ +void +ExecBatchInsertColStoreTuples(int ntuples, HeapTuple *tuples, EState *estate) +{ + ResultRelInfo *resultRelInfo; + TupleDesc tupdesc; + ListCell *l; + Datum **lvalues; + bool **lisnull; + MemoryContext tmpcxt; + MemoryContext oldcxt; + + tmpcxt = AllocSetContextCreate(CurrentMemoryContext, + "BatchInsertColStore", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + oldcxt = MemoryContextSwitchTo(tmpcxt); + + /* + * Get information from the result relation info structure. + */ + resultRelInfo = estate->es_result_relation_info; + tupdesc = RelationGetDescr(resultRelInfo->ri_RelationDesc); + + Assert(tupdesc->tdattorder == ATTRORDER_OFFHEAPATTRS); + + /* + * Allocate a pointer for each tuple. This is required by the + * ExecColumnStoreBatchInsert API function + */ + lvalues = (Datum **) palloc(sizeof(Datum *) * ntuples); + lisnull = (bool **) palloc(sizeof(bool *) * ntuples); + + foreach (l, resultRelInfo->ri_ColumnStoreHandler) + { + ColumnStoreHandler *handler = (ColumnStoreHandler *) lfirst(l); + ExecColumnStoreBatchInsert_function batchInsert; + Datum *values; + bool *isnull; + int cstore_natts; + int tup; + + batchInsert = handler->csh_ColumnStoreRoutine->ExecColumnStoreBatchInsert; + + if (batchInsert == NULL) + elog(ERROR, "BatchInsert routine not supplied by column store AM"); + + /* The column store has an extra column for the heaptid */ + cstore_natts = handler->csh_NumColumnStoreAttrs + 1; + + /* + * To save performing a palloc once for each tuple, we'll just allocate + * all in one giant block. + */ + values = (Datum *) palloc(sizeof(Datum *) * cstore_natts * ntuples); + isnull = (bool *) palloc(sizeof(bool *) * cstore_natts * ntuples); + + for (tup = 0; tup < ntuples; tup++) + { + int i; + + /* + * set lvalues and lisnull to point to the first attribute of each + * tuple. + */ + lvalues[tup] = &values[tup * cstore_natts]; + lisnull[tup] = &isnull[tup * cstore_natts]; + + /* Assign the ctid of the heap to the colstore's heaptid column */ + lvalues[tup][0] = PointerGetDatum(&tuples[tup]->t_self); + lisnull[tup][0] = false; + + for (i = 1; i < cstore_natts; i++) + { + AttrNumber attnum = handler->csh_KeyAttrNumbers[i - 1]; + lvalues[tup][i] = heap_getlogattr(tuples[tup], attnum, tupdesc, &lisnull[tup][i]); + } + } + + /* finally call the API function to store the tuples */ + batchInsert(handler, ntuples, lvalues, lisnull, estate->es_output_cid); + + pfree(values); + pfree(isnull); + } + + pfree(lvalues); + pfree(lisnull); + + MemoryContextSwitchTo(oldcxt); + MemoryContextDelete(tmpcxt); +} + +void +ExecTruncateColumnStores(Relation rel) +{ + ListCell *l; + List *cstore_oids = RelationGetColStoreList(rel); + + foreach(l, cstore_oids) + { + Oid oid = lfirst_oid(l); + Relation colstorerel = heap_open(oid, AccessExclusiveLock); + ColumnStoreRoutine *routine; + ExecColumnStoreTruncate_function csttruncate; + + routine = GetColumnStoreRoutineForRelation(colstorerel, false); + + csttruncate = routine->ExecColumnStoreTruncate; + if (csttruncate == NULL) + elog(ERROR, "Truncate routine not provided by column store AM"); + + csttruncate(colstorerel); + + heap_close(colstorerel, AccessExclusiveLock); + } +} diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 9f2af6d..c8c8a45 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -1475,6 +1475,7 @@ ExecEndPlan(PlanState *planstate, EState *estate) { /* Close indices and then the relation itself */ ExecCloseIndices(resultRelInfo); + ExecCloseColumnStores(resultRelInfo); heap_close(resultRelInfo->ri_RelationDesc, NoLock); resultRelInfo++; } @@ -1487,6 +1488,7 @@ ExecEndPlan(PlanState *planstate, EState *estate) resultRelInfo = (ResultRelInfo *) lfirst(l); /* Close indices and then the relation itself */ ExecCloseIndices(resultRelInfo); + ExecCloseColumnStores(resultRelInfo); heap_close(resultRelInfo->ri_RelationDesc, NoLock); } @@ -2904,6 +2906,7 @@ EvalPlanQualEnd(EPQState *epqstate) /* Close indices and then the relation itself */ ExecCloseIndices(resultRelInfo); + ExecCloseColumnStores(resultRelInfo); heap_close(resultRelInfo->ri_RelationDesc, NoLock); } diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index 6f5c554..eebb623 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -84,6 +84,7 @@ #include "executor/nodeBitmapHeapscan.h" #include "executor/nodeBitmapIndexscan.h" #include "executor/nodeBitmapOr.h" +#include "executor/nodeColumnStorescan.h" #include "executor/nodeCtescan.h" #include "executor/nodeCustom.h" #include "executor/nodeForeignscan.h" @@ -258,6 +259,12 @@ ExecInitNode(Plan *node, EState *estate, int eflags) estate, eflags); break; + case T_ColumnStoreScan: + result = (PlanState *) ExecInitColumnStoreScan( + (ColumnStoreScan *) node, + estate, eflags); + break; + /* * join nodes */ @@ -469,6 +476,10 @@ ExecProcNode(PlanState *node) result = ExecCustomScan((CustomScanState *) node); break; + case T_ColumnStoreScanState: + result = ExecColumnStoreScan((ColumnStoreScanState *) node); + break; + /* * join nodes */ @@ -721,6 +732,10 @@ ExecEndNode(PlanState *node) ExecEndCustomScan((CustomScanState *) node); break; + case T_ColumnStoreScanState: + ExecEndColumnStoreScan((ColumnStoreScanState *) node); + break; + /* * join nodes */ diff --git a/src/backend/executor/nodeColumnStorescan.c b/src/backend/executor/nodeColumnStorescan.c new file mode 100644 index 0000000..c2491fd --- /dev/null +++ b/src/backend/executor/nodeColumnStorescan.c @@ -0,0 +1,227 @@ +/*------------------------------------------------------------------------- + * + * nodeColumnStorescan.c + * Routines to handle column store scan nodes. + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/executor/nodeColumnStorescan.c + * + *------------------------------------------------------------------------- + */ +/* + * INTERFACE ROUTINES + * ExecColumnStoreScan - sequentially scans a column store. + * ExecInitColumnStoreScan - creates and initializes the scan node. + * ExecEndColumnStoreScan - releases any storage allocated. + * + */ +#include "postgres.h" + +#include "colstore/colstoreapi.h" +#include "executor/executor.h" +#include "executor/nodeColumnStorescan.h" +#include "utils/rel.h" +#include "utils/relcache.h" +#include "miscadmin.h" + +/* ---------------------------------------------------------------- + * ExecColumnStoreScan + * and static support routines + * ---------------------------------------------------------------- + */ +static TupleTableSlot * +ColumnStoreNext(ColumnStoreScanState *node) +{ + TupleTableSlot *slot; + ExecColumnStoreScanNext_function cstscan; + + cstscan = node->csthandler->csh_ColumnStoreRoutine->ExecColumnStoreScanNext; + if (cstscan == NULL) + elog(ERROR, "Scan routine not provided by column store AM"); + + slot = node->ss.ss_ScanTupleSlot; + slot = cstscan(node->csthandler, slot); + + return slot; +} + +static bool +ColumnStoreRecheck(ColumnStoreScanState *node) +{ + /* are there specific conditions to recheck? */ + return true; +} + +TupleTableSlot * +ExecColumnStoreScan(ColumnStoreScanState *node) +{ + return ExecScan(&node->ss, + (ExecScanAccessMtd) ColumnStoreNext, + (ExecScanRecheckMtd) ColumnStoreRecheck); +} + +/* ---------------------------------------------------------------- + * ExecInitColumnStoreScan + * ---------------------------------------------------------------- + */ +ColumnStoreScanState * +ExecInitColumnStoreScan(ColumnStoreScan *node, EState *estate, int eflags) +{ + ColumnStoreScanState *colscanstate; + Relation currentRelation; + Index scanrelid = node->scan.scanrelid; + + Assert(outerPlan(node) == NULL); + Assert(innerPlan(node) == NULL); + + /* + * create state structure + */ + colscanstate = makeNode(ColumnStoreScanState); + colscanstate->ss.ps.plan = (Plan *) node; + colscanstate->ss.ps.state = estate; + + /* don't set flags because the suport routines are not implemented yet */ +#if 0 + colscanstate->eflags = (eflags & (EXEC_FLAG_REWIND | + EXEC_FLAG_BACKWARD | + EXEC_FLAG_MARK)); +#endif + + currentRelation = ExecOpenScanRelation(estate, node->scan.scanrelid, eflags); + colscanstate->ss.ss_currentRelation = currentRelation; + colscanstate->csthandler = + BuildColumnStoreHandler(currentRelation, false, + AccessShareLock, estate->es_snapshot); + + /* + * Miscellaneous initialization + * + * create expression context for node + */ + ExecAssignExprContext(estate, &colscanstate->ss.ps); + + /* + * initialize child expressions + */ + colscanstate->ss.ps.targetlist = (List *) + ExecInitExpr((Expr *) node->scan.plan.targetlist, &colscanstate->ss.ps); + colscanstate->ss.ps.qual = (List *) + ExecInitExpr((Expr *) node->scan.plan.qual, &colscanstate->ss.ps); + + /* + * tuple table initialization + */ + ExecInitResultTupleSlot(estate, &colscanstate->ss.ps); + ExecInitScanTupleSlot(estate, &colscanstate->ss); + + /* + * initialize scan relation + */ + ExecAssignScanType(&colscanstate->ss, RelationGetDescr(currentRelation)); + colscanstate->ss.ps.ps_TupFromTlist = false; + + /* + * Initialize result tuple type and projection info. + */ + ExecAssignResultTypeFromTL(&colscanstate->ss.ps); + ExecAssignScanProjectionInfoWithVarno(&colscanstate->ss, scanrelid); + + return colscanstate; +} + +/* ---------------------------------------------------------------- + * ExecEndColumnStoreScan + * ---------------------------------------------------------------- + */ +void +ExecEndColumnStoreScan(ColumnStoreScanState *node) +{ + Relation relation; + ExecColumnStoreEndScan_function cstendscan; + + cstendscan = node->csthandler->csh_ColumnStoreRoutine->ExecColumnStoreEndScan; + if (cstendscan == NULL) + elog(ERROR, "EndScan routine not provided by column store AM"); + + /* + * get information from node + */ + relation = node->ss.ss_currentRelation; + + /* Call the ExecColumnStoreEndScan API function */ + cstendscan(node->csthandler); + + /* + * clean out the tuple table + */ + ExecClearTuple(node->ss.ss_ScanTupleSlot); + + ExecCloseScanRelation(relation); +} + +/* ---------------------------------------------------------------- + * ExecColumnStoreScanMarkPos + * ---------------------------------------------------------------- + */ +void +ExecColumnStoreScanMarkPos(ColumnStoreScanState *node) +{ + ExecColumnStoreMarkPos_function markpos; + + Assert(node->eflags & EXEC_FLAG_MARK); + + markpos = node->csthandler->csh_ColumnStoreRoutine->ExecColumnStoreMarkPos; + + if (!markpos) + elog(ERROR, "MarkPos routine not supplied by column store AM"); + + markpos(node->csthandler); +} + +/* ---------------------------------------------------------------- + * ExecColumnStoreScanRestrPos + * ---------------------------------------------------------------- + */ +void +ExecColumnStoreScanRestrPos(ColumnStoreScanState *node) +{ + ExecColumnStoreRestrPos_function restore; + + Assert(node->eflags & EXEC_FLAG_MARK); + + restore = node->csthandler->csh_ColumnStoreRoutine->ExecColumnStoreRestrPos; + + if (!restore) + elog(ERROR, "RestrPos routine not supplied by column store AM"); + + restore(node->csthandler); +} + +/* ---------------------------------------------------------------- + * ExecReScanColumnStoreScan + * + * Rescans the col store relation. + * ---------------------------------------------------------------- + */ +void +ExecReScanColumnStoreScan(ColumnStoreScanState *node) +{ + ColumnStoreHandler *handler = node->csthandler; + ExecColumnStoreRescan_function rescan; + + rescan = handler->csh_ColumnStoreRoutine->ExecColumnStoreRescan; + + ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); + + if (rescan == NULL) + elog(ERROR, "ReScan routine not supplied by column store AM"); + + rescan(node->csthandler); + + ExecScanReScan(&node->ss); +} diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 9db4c91..5ec057f 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -39,6 +39,7 @@ #include "access/htup_details.h" #include "access/xact.h" +#include "catalog/colstore.h" #include "commands/trigger.h" #include "executor/executor.h" #include "executor/nodeModifyTable.h" @@ -402,11 +403,56 @@ ExecInsert(ModifyTableState *mtstate, specToken = SpeculativeInsertionLockAcquire(GetCurrentTransactionId()); HeapTupleHeaderSetSpeculativeToken(tuple->t_data, specToken); - /* insert the tuple, with the speculative token */ - newId = heap_insert(resultRelationDesc, tuple, - estate->es_output_cid, - HEAP_INSERT_SPECULATIVE, - NULL); + /* + * insert the tuple, with the speculative token + * + * Note: heap_insert returns the tid (location) of the new tuple in + * the t_self field. + * + * We need to remove the columns that are stored in the column store + * from the descriptor and heap tuple, so that we only store the heap + * part using heap_insert. We'll create a new tuple descriptor with + * only the heap attributes, and create a small 'heap tuple' matching + * the descriptor. + */ + + if (resultRelationDesc->rd_att->tdattorder == ATTRORDER_OFFHEAPATTRS) + { + TupleDesc tupDesc = resultRelationDesc->rd_att; + int nattrs = tupDesc->natts; + HeapTuple heaptuple; + Datum *values; + bool *isnull; + int attr; + + /* + * Tuple requires translation from the logical tuple order, + * into on-disk physical tuple order. Perhaps there is a better + * way to do this, maybe we could delay forming the tuple and + * only ever form a physical tuple? + */ + values = (Datum *) palloc(nattrs * sizeof(Datum)); + isnull = (bool *) palloc(nattrs * sizeof(bool)); + + for (attr = 0; attr < nattrs; attr++) + values[attr] = heap_getlogattr(tuple, attr + 1, tupDesc, &isnull[attr]); + + heaptuple = heap_form_tuple(tupDesc, values, isnull); + TupleDescCacheReset(tupDesc); + + newId = heap_insert(resultRelationDesc, heaptuple, + estate->es_output_cid, + 0, NULL); + ItemPointerCopy(&heaptuple->t_self, &tuple->t_self); + + ExecInsertColStoreTuples(tuple, estate); + TupleDescCacheReset(tupDesc); /* XXX yuck, but seems to be needed */ + } + else + newId = heap_insert(resultRelationDesc, tuple, + estate->es_output_cid, + HEAP_INSERT_SPECULATIVE, + NULL); /* insert index entries for tuple */ recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), @@ -443,15 +489,43 @@ ExecInsert(ModifyTableState *mtstate, } else { - /* - * insert the tuple normally. - * - * Note: heap_insert returns the tid (location) of the new tuple - * in the t_self field. - */ - newId = heap_insert(resultRelationDesc, tuple, - estate->es_output_cid, - 0, NULL); + TupleDesc tupDesc = resultRelationDesc->rd_att; + + if (tupDesc->tdattorder == ATTRORDER_OFFHEAPATTRS) + { + int nattrs = tupDesc->natts; + HeapTuple heaptuple; + Datum *values; + bool *isnull; + int attr; + + /* + * Tuple requires translation from the logical tuple order, + * into on-disk physical tuple order. Perhaps there is a better + * way to do this, maybe we could delay forming the tuple and + * only ever form a physical tuple? + */ + values = (Datum *) palloc(nattrs * sizeof(Datum)); + isnull = (bool *) palloc(nattrs * sizeof(bool)); + + for (attr = 0; attr < nattrs; attr++) + values[attr] = heap_getlogattr(tuple, attr + 1, tupDesc, &isnull[attr]); + + heaptuple = heap_form_tuple(tupDesc, values, isnull); + TupleDescCacheReset(tupDesc); + + newId = heap_insert(resultRelationDesc, heaptuple, + estate->es_output_cid, + 0, NULL); + ItemPointerCopy(&heaptuple->t_self, &tuple->t_self); + + ExecInsertColStoreTuples(tuple, estate); + TupleDescCacheReset(tupDesc); /* XXX yuck, but seems to be needed */ + } + else + newId = heap_insert(resultRelationDesc, tuple, + estate->es_output_cid, + 0, NULL); /* insert index entries for tuple */ if (resultRelInfo->ri_NumIndices > 0) @@ -880,12 +954,47 @@ lreplace:; * can't-serialize error if not. This is a special-case behavior * needed for referential integrity updates in transaction-snapshot * mode transactions. + * + * We need to remove the columns that are stored in the column store + * from the descriptor and heap tuple, so that we only store the heap + * part using heap_insert. We'll create a new tuple descriptor with + * only the heap attributes, and create a small 'heap tuple' matching + * the descriptor. + * + * FIXME This is just temporary solution, a bit dirty. Needs to be + * done properly (moved to methods, possibly applied to other + * places, etc.). */ - result = heap_update(resultRelationDesc, tupleid, tuple, - estate->es_output_cid, - estate->es_crosscheck_snapshot, - true /* wait for commit */ , - &hufd, &lockmode); + if (resultRelInfo->ri_ColumnStoreHandler != NIL) + { + HeapTuple heaptuple; + TupleDesc heapdesc; + TupleDesc fulldesc; + + heaptuple = FilterHeapTuple(resultRelInfo, tuple, &heapdesc); + + fulldesc = resultRelationDesc->rd_att; + resultRelationDesc->rd_att = heapdesc; + + result = heap_update(resultRelationDesc, tupleid, heaptuple, + estate->es_output_cid, + estate->es_crosscheck_snapshot, + true /* wait for commit */ , + &hufd, &lockmode); + + resultRelationDesc->rd_att = fulldesc; + + heap_freetuple(heaptuple); + FreeTupleDesc(heapdesc); + } + else + result = heap_update(resultRelationDesc, tupleid, tuple, + estate->es_output_cid, + estate->es_crosscheck_snapshot, + true /* wait for commit */ , + &hufd, &lockmode); + + switch (result) { case HeapTupleSelfUpdated: @@ -966,16 +1075,20 @@ lreplace:; */ /* - * insert index entries for tuple + * insert index and column store entries for tuple * * Note: heap_update returns the tid (location) of the new tuple in * the t_self field. * - * If it's a HOT update, we mustn't insert new index entries. + * If it's a HOT update, we mustn't insert new index and column store + * entries. */ if (resultRelInfo->ri_NumIndices > 0 && !HeapTupleIsHeapOnly(tuple)) recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self), estate, false, NULL, NIL); + + if (resultRelInfo->ri_ColumnStoreHandler != NIL && !HeapTupleIsHeapOnly(tuple)) + ExecInsertColStoreTuples(tuple, estate); } if (canSetTag) @@ -1555,6 +1668,9 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags) resultRelInfo->ri_IndexRelationDescs == NULL) ExecOpenIndices(resultRelInfo, mtstate->mt_onconflict != ONCONFLICT_NONE); + /* TODO should use relhascolstore just like indexes*/ + ExecOpenColumnStores(resultRelInfo); + /* Now init the plan for this result rel */ estate->es_result_relation_info = resultRelInfo; mtstate->mt_plans[i] = ExecInitNode(subplan, estate, eflags); diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 4cf14b6..a14f4a7 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -690,6 +690,19 @@ _copyCustomScan(const CustomScan *from) } /* + * _copyColumnStoreScan + */ +static ColumnStoreScan * +_copyColumnStoreScan(const ColumnStoreScan *from) +{ + ColumnStoreScan *newnode = makeNode(ColumnStoreScan); + + CopyPlanFields((const Plan *) from, (Plan *) newnode); + + return newnode; +} + +/* * CopyJoinFields * * This function copies the fields of the Join node. It is used by @@ -812,7 +825,6 @@ _copyMaterial(const Material *from) return newnode; } - /* * _copySort */ @@ -2085,6 +2097,21 @@ _copyAppendRelInfo(const AppendRelInfo *from) } /* + * _copyColstoreRelInfo + */ +static ColstoreRelInfo * +_copyColstoreRelInfo(const ColstoreRelInfo *from) +{ + ColstoreRelInfo *newnode = makeNode(ColstoreRelInfo); + + COPY_SCALAR_FIELD(parent_relid); + COPY_SCALAR_FIELD(child_relid); + COPY_SCALAR_FIELD(child_oid); + + return newnode; +} + +/* * _copyPlaceHolderInfo */ static PlaceHolderInfo * @@ -2115,6 +2142,7 @@ _copyRangeTblEntry(const RangeTblEntry *from) COPY_SCALAR_FIELD(rtekind); COPY_SCALAR_FIELD(relid); COPY_SCALAR_FIELD(relkind); + COPY_SCALAR_FIELD(relhascstore); COPY_NODE_FIELD(tablesample); COPY_NODE_FIELD(subquery); COPY_SCALAR_FIELD(security_barrier); @@ -2568,6 +2596,21 @@ _copyCollateClause(const CollateClause *from) return newnode; } +static ColumnStoreClause * +_copyColumnStoreClause(const ColumnStoreClause *from) +{ + ColumnStoreClause *newnode = makeNode(ColumnStoreClause); + + COPY_STRING_FIELD(name); + COPY_STRING_FIELD(storetype); + COPY_NODE_FIELD(columns); + COPY_NODE_FIELD(options); + COPY_LOCATION_FIELD(location); + COPY_STRING_FIELD(tablespacename); + + return newnode; +} + static IndexElem * _copyIndexElem(const IndexElem *from) { @@ -2600,6 +2643,7 @@ _copyColumnDef(const ColumnDef *from) COPY_NODE_FIELD(cooked_default); COPY_NODE_FIELD(collClause); COPY_SCALAR_FIELD(collOid); + COPY_NODE_FIELD(cstoreClause); COPY_NODE_FIELD(constraints); COPY_NODE_FIELD(fdwoptions); COPY_LOCATION_FIELD(location); @@ -3809,6 +3853,17 @@ _copyImportForeignSchemaStmt(const ImportForeignSchemaStmt *from) return newnode; } +static CreateColumnStoreAMStmt * +_copyCreateColumnStoreAMStmt(const CreateColumnStoreAMStmt *from) +{ + CreateColumnStoreAMStmt *newnode = makeNode(CreateColumnStoreAMStmt); + + COPY_STRING_FIELD(cstamname); + COPY_NODE_FIELD(func_options); + + return newnode; +} + static CreateTransformStmt * _copyCreateTransformStmt(const CreateTransformStmt *from) { @@ -4306,6 +4361,9 @@ copyObject(const void *from) case T_Material: retval = _copyMaterial(from); break; + case T_ColumnStoreScan: + retval = _copyColumnStoreScan(from); + break; case T_Sort: retval = _copySort(from); break; @@ -4509,6 +4567,9 @@ copyObject(const void *from) case T_AppendRelInfo: retval = _copyAppendRelInfo(from); break; + case T_ColstoreRelInfo: + retval = _copyColstoreRelInfo(from); + break; case T_PlaceHolderInfo: retval = _copyPlaceHolderInfo(from); break; @@ -4783,6 +4844,9 @@ copyObject(const void *from) case T_ImportForeignSchemaStmt: retval = _copyImportForeignSchemaStmt(from); break; + case T_CreateColumnStoreAMStmt: + retval = _copyCreateColumnStoreAMStmt(from); + break; case T_CreateTransformStmt: retval = _copyCreateTransformStmt(from); break; @@ -4897,6 +4961,9 @@ copyObject(const void *from) case T_CollateClause: retval = _copyCollateClause(from); break; + case T_ColumnStoreClause: + retval = _copyColumnStoreClause(from); + break; case T_SortBy: retval = _copySortBy(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index a13d831..4b7698d 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -859,6 +859,16 @@ _equalAppendRelInfo(const AppendRelInfo *a, const AppendRelInfo *b) } static bool +_equalColstoreRelInfo(const ColstoreRelInfo *a, const ColstoreRelInfo *b) +{ + COMPARE_SCALAR_FIELD(parent_relid); + COMPARE_SCALAR_FIELD(child_relid); + COMPARE_SCALAR_FIELD(child_oid); + + return true; +} + +static bool _equalPlaceHolderInfo(const PlaceHolderInfo *a, const PlaceHolderInfo *b) { COMPARE_SCALAR_FIELD(phid); @@ -1823,6 +1833,15 @@ _equalImportForeignSchemaStmt(const ImportForeignSchemaStmt *a, const ImportFore } static bool +_equalCreateColumnStoreAMStmt(const CreateColumnStoreAMStmt *a, const CreateColumnStoreAMStmt *b) +{ + COMPARE_STRING_FIELD(cstamname); + COMPARE_NODE_FIELD(func_options); + + return true; +} + +static bool _equalCreateTransformStmt(const CreateTransformStmt *a, const CreateTransformStmt *b) { COMPARE_SCALAR_FIELD(replace); @@ -2233,6 +2252,19 @@ _equalCollateClause(const CollateClause *a, const CollateClause *b) } static bool +_equalColumnStoreClause(const ColumnStoreClause *a, const ColumnStoreClause *b) +{ + COMPARE_STRING_FIELD(name); + COMPARE_STRING_FIELD(storetype); + COMPARE_NODE_FIELD(columns); + COMPARE_NODE_FIELD(options); + COMPARE_LOCATION_FIELD(location); + COMPARE_STRING_FIELD(tablespacename); + + return true; +} + +static bool _equalSortBy(const SortBy *a, const SortBy *b) { COMPARE_NODE_FIELD(node); @@ -2322,6 +2354,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b) COMPARE_NODE_FIELD(cooked_default); COMPARE_NODE_FIELD(collClause); COMPARE_SCALAR_FIELD(collOid); + COMPARE_NODE_FIELD(cstoreClause); COMPARE_NODE_FIELD(constraints); COMPARE_NODE_FIELD(fdwoptions); COMPARE_LOCATION_FIELD(location); @@ -2855,6 +2888,9 @@ equal(const void *a, const void *b) case T_AppendRelInfo: retval = _equalAppendRelInfo(a, b); break; + case T_ColstoreRelInfo: + retval = _equalColstoreRelInfo(a, b); + break; case T_PlaceHolderInfo: retval = _equalPlaceHolderInfo(a, b); break; @@ -3116,6 +3152,9 @@ equal(const void *a, const void *b) case T_ImportForeignSchemaStmt: retval = _equalImportForeignSchemaStmt(a, b); break; + case T_CreateColumnStoreAMStmt: + retval = _equalCreateColumnStoreAMStmt(a, b); + break; case T_CreateTransformStmt: retval = _equalCreateTransformStmt(a, b); break; @@ -3230,6 +3269,9 @@ equal(const void *a, const void *b) case T_CollateClause: retval = _equalCollateClause(a, b); break; + case T_ColumnStoreClause: + retval = _equalColumnStoreClause(a, b); + break; case T_SortBy: retval = _equalSortBy(a, b); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index a11cb9f..90d1cf3 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -1999,6 +1999,9 @@ expression_tree_walker(Node *node, return true; } break; + case T_ColstoreRelInfo: + /* nothing to do at present */ + break; case T_PlaceHolderInfo: return walker(((PlaceHolderInfo *) node)->ph_var, context); case T_RangeTblFunction: @@ -2774,6 +2777,15 @@ expression_tree_mutator(Node *node, return (Node *) newnode; } break; + case T_ColstoreRelInfo: + { + ColstoreRelInfo *cstinfo = (ColstoreRelInfo *) node; + ColstoreRelInfo *newnode; + + FLATCOPY(newnode, cstinfo, ColstoreRelInfo); + return (Node *) newnode; + } + break; case T_PlaceHolderInfo: { PlaceHolderInfo *phinfo = (PlaceHolderInfo *) node; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 7169d46..04cf41b 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -621,6 +621,14 @@ _outCustomScan(StringInfo str, const CustomScan *node) } static void +_outColumnStoreScan(StringInfo str, const ColumnStoreScan *node) +{ + WRITE_NODE_TYPE("COLUMNSTORESCAN"); + + _outScanInfo(str, (const Scan *) node); +} + +static void _outJoin(StringInfo str, const Join *node) { WRITE_NODE_TYPE("JOIN"); @@ -1704,6 +1712,15 @@ _outCustomPath(StringInfo str, const CustomPath *node) } static void +_outColumnStoreScanPath(StringInfo str, const ColumnStoreScanPath *node) +{ + WRITE_NODE_TYPE("COLUMNSTORESCANPATH"); + + _outPathInfo(str, (const Path *) node); + WRITE_NODE_FIELD(colstore); +} + +static void _outAppendPath(StringInfo str, const AppendPath *node) { WRITE_NODE_TYPE("APPENDPATH"); @@ -1848,6 +1865,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node) WRITE_NODE_FIELD(full_join_clauses); WRITE_NODE_FIELD(join_info_list); WRITE_NODE_FIELD(append_rel_list); + WRITE_NODE_FIELD(colstore_rel_list); WRITE_NODE_FIELD(rowMarks); WRITE_NODE_FIELD(placeholder_list); WRITE_NODE_FIELD(query_pathkeys); @@ -1901,6 +1919,7 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node) WRITE_NODE_FIELD(lateral_vars); WRITE_BITMAPSET_FIELD(lateral_referencers); WRITE_NODE_FIELD(indexlist); + WRITE_NODE_FIELD(cstlist); WRITE_UINT_FIELD(pages); WRITE_FLOAT_FIELD(tuples, "%.0f"); WRITE_FLOAT_FIELD(allvisfrac, "%.6f"); @@ -1939,6 +1958,23 @@ _outIndexOptInfo(StringInfo str, const IndexOptInfo *node) } static void +_outColumnStoreOptInfo(StringInfo str, const ColumnStoreOptInfo *node) +{ + WRITE_NODE_TYPE("COLUMNSTOREOPTINFO"); + + /* NB: this isn't a complete set of fields */ + WRITE_OID_FIELD(colstoreoid); + + /* Do NOT print rel field, else infinite recursion */ + WRITE_UINT_FIELD(pages); + WRITE_FLOAT_FIELD(tuples, "%.0f"); + WRITE_INT_FIELD(ncolumns); + /* array fields aren't really worth the trouble to print */ + WRITE_OID_FIELD(cstam); + /* we don't bother with fields copied from the pg_am entry */ +} + +static void _outEquivalenceClass(StringInfo str, const EquivalenceClass *node) { /* @@ -2070,6 +2106,16 @@ _outAppendRelInfo(StringInfo str, const AppendRelInfo *node) } static void +_outColstoreRelInfo(StringInfo str, const ColstoreRelInfo *node) +{ + WRITE_NODE_TYPE("COLSTORERELINFO"); + + WRITE_UINT_FIELD(parent_relid); + WRITE_UINT_FIELD(child_relid); + WRITE_OID_FIELD(child_oid); +} + +static void _outPlaceHolderInfo(StringInfo str, const PlaceHolderInfo *node) { WRITE_NODE_TYPE("PLACEHOLDERINFO"); @@ -2305,6 +2351,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node) WRITE_NODE_FIELD(collClause); WRITE_OID_FIELD(collOid); WRITE_NODE_FIELD(constraints); + WRITE_NODE_FIELD(cstoreClause); WRITE_NODE_FIELD(fdwoptions); WRITE_LOCATION_FIELD(location); } @@ -2345,6 +2392,19 @@ _outCollateClause(StringInfo str, const CollateClause *node) } static void +_outColumnStoreClause(StringInfo str, const ColumnStoreClause *node) +{ + WRITE_NODE_TYPE("COLUMNSTORECLAUSE"); + + WRITE_STRING_FIELD(name); + WRITE_STRING_FIELD(storetype); + WRITE_NODE_FIELD(columns); + WRITE_NODE_FIELD(options); + WRITE_LOCATION_FIELD(location); + WRITE_STRING_FIELD(tablespacename); +} + +static void _outIndexElem(StringInfo str, const IndexElem *node) { WRITE_NODE_TYPE("INDEXELEM"); @@ -3068,6 +3128,9 @@ _outNode(StringInfo str, const void *obj) case T_CustomScan: _outCustomScan(str, obj); break; + case T_ColumnStoreScan: + _outColumnStoreScan(str, obj); + break; case T_Join: _outJoin(str, obj); break; @@ -3287,6 +3350,9 @@ _outNode(StringInfo str, const void *obj) case T_CustomPath: _outCustomPath(str, obj); break; + case T_ColumnStoreScanPath: + _outColumnStoreScanPath(str, obj); + break; case T_AppendPath: _outAppendPath(str, obj); break; @@ -3350,6 +3416,9 @@ _outNode(StringInfo str, const void *obj) case T_AppendRelInfo: _outAppendRelInfo(str, obj); break; + case T_ColstoreRelInfo: + _outColstoreRelInfo(str, obj); + break; case T_PlaceHolderInfo: _outPlaceHolderInfo(str, obj); break; @@ -3359,7 +3428,9 @@ _outNode(StringInfo str, const void *obj) case T_PlannerParamItem: _outPlannerParamItem(str, obj); break; - + case T_ColumnStoreOptInfo: + _outColumnStoreOptInfo(str, obj); + break; case T_CreateStmt: _outCreateStmt(str, obj); break; @@ -3393,6 +3464,9 @@ _outNode(StringInfo str, const void *obj) case T_CollateClause: _outCollateClause(str, obj); break; + case T_ColumnStoreClause: + _outColumnStoreClause(str, obj); + break; case T_IndexElem: _outIndexElem(str, obj); break; diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c index 4516cd3..2bc5d0c 100644 --- a/src/backend/optimizer/path/allpaths.c +++ b/src/backend/optimizer/path/allpaths.c @@ -77,6 +77,10 @@ static void set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, static bool function_rte_parallel_ok(RangeTblEntry *rte); static void set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); +static void set_colstore_rel_size(PlannerInfo *root, RelOptInfo *rel, + RangeTblEntry *rte); +static void set_colstore_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, + RangeTblEntry *rte); static void set_tablesample_rel_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte); static void set_tablesample_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, @@ -343,6 +347,11 @@ set_rel_size(PlannerInfo *root, RelOptInfo *rel, /* Sampled relation */ set_tablesample_rel_size(root, rel, rte); } + else if (rte->relkind == RELKIND_COLUMN_STORE) + { + /* column store */ + set_colstore_rel_size(root, rel, rte); + } else { /* Plain relation */ @@ -420,6 +429,11 @@ set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, /* Sampled relation */ set_tablesample_rel_pathlist(root, rel, rte); } + else if (rte->relkind == RELKIND_COLUMN_STORE) + { + /* column store */ + set_colstore_rel_pathlist(root, rel, rte); + } else { /* Plain relation */ @@ -665,6 +679,26 @@ set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) create_tidscan_paths(root, rel); } +static void +set_colstore_rel_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) +{ + /* XXX temporary (?) hack */ + set_plain_rel_size(root, rel, rte); +} + +static void +set_colstore_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) +{ + /* consider column store scan path */ + add_path(rel, create_colstore_scan_path(root, rel)); + + /* + * XXX The other path(s) we should consider is a column scan that uses some + * qual from WHERE etc to produce CTIDs, which parametrizes the heap scan. + * We don't have that yet. + */ +} + /* * set_tablesample_rel_size * Set size estimates for a sampled relation @@ -2731,6 +2765,9 @@ print_path(PlannerInfo *root, Path *path, int indent) case T_WorkTableScan: ptype = "WorkTableScan"; break; + case T_ColumnStoreScan: + ptype = "ColumnStoreScan"; + break; default: ptype = "???Path"; break; diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 990486c..7b07cd4 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -77,6 +77,7 @@ #include "access/htup_details.h" #include "access/tsmapi.h" +#include "catalog/pg_class.h" #include "executor/executor.h" #include "executor/nodeHash.h" #include "miscadmin.h" @@ -341,6 +342,37 @@ cost_gather(GatherPath *path, PlannerInfo *root, } /* + * cost_colstore_scan + * Determines and returns the cost of scanning a column store. + * + * 'baserel' is the column store relation to be scanned + * 'param_info' is the ParamPathInfo if this is a parameterized path, else NULL + */ +void +cost_colstore_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, + ParamPathInfo *param_info) +{ + Cost run_cost = 0; + Cost startup_cost = 0; + double spc_seq_page_cost; + + /* Should only be applied to base relations */ + Assert(baserel->relid > 0); + Assert(baserel->rtekind == RTE_RELATION); + + path->rows = baserel->rows; + + get_tablespace_page_costs(baserel->reltablespace, + NULL, + &spc_seq_page_cost); + + run_cost = spc_seq_page_cost * baserel->pages; + + path->startup_cost = startup_cost; + path->total_cost = startup_cost + run_cost; +} + +/* * cost_index * Determines and returns the cost of scanning a relation using an index. * @@ -1036,6 +1068,7 @@ cost_tidscan(Path *path, PlannerInfo *root, int ntuples; ListCell *l; double spc_random_page_cost; + int paramrelid; /* Should only be applied to base relations */ Assert(baserel->relid > 0); @@ -1094,11 +1127,33 @@ cost_tidscan(Path *path, PlannerInfo *root, */ cost_qual_eval(&tid_qual_cost, tidquals, root); - /* fetch estimated page cost for tablespace containing table */ - get_tablespace_page_costs(baserel->reltablespace, - &spc_random_page_cost, - NULL); + /* + * XXX dirty hack to reduce costs when joining to column stores. + * The current column store implementation will produce TIDs in + * order, so this is more sequential than random + */ + if (param_info && + bms_get_singleton_member(param_info->ppi_req_outer, ¶mrelid)) + { + RangeTblEntry *outerrte = root->simple_rte_array[paramrelid]; + if (outerrte->relkind == RELKIND_COLUMN_STORE) + spc_random_page_cost = seq_page_cost; + else + { + /* fetch estimated page cost for tablespace containing table */ + get_tablespace_page_costs(baserel->reltablespace, + &spc_random_page_cost, + NULL); + } + } + else + { + /* fetch estimated page cost for tablespace containing table */ + get_tablespace_page_costs(baserel->reltablespace, + &spc_random_page_cost, + NULL); + } /* disk costs --- assume each tuple on a different page */ run_cost += spc_random_page_cost * ntuples; diff --git a/src/backend/optimizer/path/tidpath.c b/src/backend/optimizer/path/tidpath.c index 1258961..6c62e45 100644 --- a/src/backend/optimizer/path/tidpath.c +++ b/src/backend/optimizer/path/tidpath.c @@ -102,10 +102,6 @@ IsTidEqualClause(OpExpr *node, int varno) if (exprType(other) != TIDOID) return false; /* probably can't happen */ - /* The other argument must be a pseudoconstant */ - if (!is_pseudo_constant_clause(other)) - return false; - return true; /* success */ } @@ -264,4 +260,44 @@ create_tidscan_paths(PlannerInfo *root, RelOptInfo *rel) if (tidquals) add_path(rel, (Path *) create_tidscan_path(root, rel, tidquals, required_outer)); + + /* + * If this table has column stores, scan them to add additional paths for + * TidScans parameterized by the column store scans. + */ + if (rel->cstlist != NIL) + { + ListCell *l; + + foreach(l, root->colstore_rel_list) + { + ColstoreRelInfo *cst = lfirst(l); + RelOptInfo *colstore; + List *joinquals; + Relids joinrels; + + /* colstore_rel_info contains all column stores; ignore others */ + if (cst->parent_relid != rel->relid) + continue; + + /* ignore colstores removed from join tree */ + colstore = root->simple_rel_array[cst->child_relid]; + if (colstore->reloptkind != RELOPT_BASEREL) + continue; + + joinrels = bms_union(colstore->relids, rel->relids); + joinquals = generate_join_implied_equalities(root, joinrels, + colstore->relids, rel); + + tidquals = TidQualFromRestrictinfo(joinquals, rel->relid); + if (tidquals) + { + required_outer = bms_add_member(NULL, cst->child_relid); + add_path(rel, + (Path *) create_tidscan_path(root, rel, tidquals, + required_outer)); + } + } + + } } diff --git a/src/backend/optimizer/plan/analyzejoins.c b/src/backend/optimizer/plan/analyzejoins.c index d188d97..e918f51 100644 --- a/src/backend/optimizer/plan/analyzejoins.c +++ b/src/backend/optimizer/plan/analyzejoins.c @@ -22,6 +22,8 @@ */ #include "postgres.h" +#include "access/sysattr.h" +#include "catalog/pg_class.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/joininfo.h" @@ -32,12 +34,36 @@ #include "utils/lsyscache.h" /* local functions */ +static bool colstore_join_is_removable(PlannerInfo *root, RelOptInfo *csrel, + Relids *joinrelids); +static inline bool colstore_attrs_used(PlannerInfo *root, Relids joinrelids, + RelOptInfo *rel); static bool join_is_removable(PlannerInfo *root, SpecialJoinInfo *sjinfo); static void remove_rel_from_query(PlannerInfo *root, int relid, Relids joinrelids); static List *remove_rel_from_joinlist(List *joinlist, int relid, int *nremoved); static Oid distinct_col_search(int colno, List *colnos, List *opids); +/* + * Find the RTI of the relation that owns colstore with given RTI. + * + * XXX this should probably be elsewhere. + */ +static Index +find_colstore_parentrelid(PlannerInfo *root, int colstoreid) +{ + ListCell *lc; + + foreach(lc, root->colstore_rel_list) + { + ColstoreRelInfo *info = (ColstoreRelInfo *) lfirst(lc); + + if (info->child_relid == colstoreid) + return info->parent_relid; + } + + return 0; /* unable to find */ +} /* * remove_useless_joins @@ -53,10 +79,37 @@ remove_useless_joins(PlannerInfo *root, List *joinlist) ListCell *lc; /* - * We are only interested in relations that are left-joined to, so we can - * scan the join_info_list to find them easily. + * Because we may have added excessive column stores to the join list due + * to join processing, we remove now those that are found to be + * unnecessary. */ restart: + foreach(lc, joinlist) + { + RangeTblRef *rtr = (RangeTblRef *) lfirst(lc); + RangeTblEntry *rte = root->simple_rte_array[rtr->rtindex]; + + if (rte->relkind == RELKIND_COLUMN_STORE) + { + RelOptInfo *csrel = find_base_rel(root, rtr->rtindex); + Relids joinrelids; + + if (colstore_join_is_removable(root, csrel, &joinrelids)) + { + int nremoved = 0; + + remove_rel_from_query(root, csrel->relid, joinrelids); + + joinlist = remove_rel_from_joinlist(joinlist, rtr->rtindex, &nremoved); + if (nremoved != 1) + elog(ERROR, "failed to find relation %d in joinlist", rtr->rtindex); + + goto restart; + } + } + } + + /* scan the join_info_list to check for LEFT JOINs which can be removed. */ foreach(lc, root->join_info_list) { SpecialJoinInfo *sjinfo = (SpecialJoinInfo *) lfirst(lc); @@ -105,6 +158,193 @@ restart: } /* + * colstore_join_is_removable + * Determines if a join to a column store may safely be removed without + * affecting the results of the query. + * + * The requirements for successful join removal here are: + * 1. No attribute of the column store is used in the query apart from on the + * join condition to the column store's parent relation. + * 2. The join condition to the column store heap must be on only the heap's + * ctid joining to the column store's heaptid column. + * + * If any other quals exist which restrict rows coming from the column store + * in any way, then we cannot be certain of matching exactly 1 row in the + * column store, therefore we cannot remove the join. + * + * Returns true if the join can safely be removed. joinrelids is set to + * the heap's relid and the column stores relid. + */ +static bool +colstore_join_is_removable(PlannerInfo *root, RelOptInfo *csrel, + Relids *joinrelids) +{ + Index colstoreparent; + ListCell *l; + + /* If the column store has any restriction quals then we can't remove */ + if (csrel->baserestrictinfo != NIL) + return false; + + /* + * We require that the only join qual be the ctid = heaptid qual. This + * will be an eclass join, but there may be joininfo items which represent + * the same condition. Here we'll make a pass over the joininfo list to + * ensire that any quals that are in there will also be found by the eclass + * scanning code below. + */ + if (csrel->joininfo != NIL) + { + foreach(l, csrel->joininfo) + { + RestrictInfo *rinfo = (RestrictInfo *) lfirst(l); + + /* XXX I'm really not sure that this is correct at all. */ + if (!rinfo->left_ec || !rinfo->right_ec) + return false; + } + } + + colstoreparent = find_colstore_parentrelid(root, csrel->relid); + + /* + * If the colstore's heap is not present in the query then we can't + * remove the join + */ + if (colstoreparent == 0) + return false; + + *joinrelids = bms_copy(csrel->relids); + *joinrelids = bms_add_member(*joinrelids, colstoreparent); + + /* + * If any attributes are required below the join, with the expection of + * heaptid, then we can't remove the join. heaptid cannot be manually + * referenced in the query to perform any other filtering as this column + * does not exist logically in the heap table, therefore an error would + * have been generated at the parser if it had. + */ + if (colstore_attrs_used(root, *joinrelids, csrel)) + return false; + + /* + * Now look over each EquivalenceClass. Here we're looking to ensure that + * the only join condition is heap.ctid = colstore.heaptid. If we discover + * any other Vars which belong to the colstore which are not the heaptid + * column, then we must abort the join removal. + */ + foreach(l, root->eq_classes) + { + EquivalenceClass *ec = (EquivalenceClass *) lfirst(l); + ListCell *l2; + bool gotheapctid = false; + bool gotcolstoreheaptid = false; + + if (ec->ec_broken || ec->ec_merged) + continue; + + /* Skip eclasses which have no members for the csrel */ + if (!bms_is_subset(*joinrelids, ec->ec_relids)) + continue; + + foreach(l2, ec->ec_members) + { + EquivalenceMember *em = (EquivalenceMember *) lfirst(l2); + Var *var = (Var *) em->em_expr; + + if (!IsA(var, Var)) + continue; + + if (var->varno == colstoreparent) + { + if (var->varattno == SelfItemPointerAttributeNumber) + gotheapctid = true; + } + else if (var->varno == csrel->relid) + { + if (var->varattno == 1) /* XXX magic number */ + gotcolstoreheaptid = true; + else + return false; + } + } + + /* + * If we didn't find either of these then we must abort the join + * removal as it means that the colstore has equivalance with something + * else apart from the heap ctid + */ + if (!gotheapctid || !gotcolstoreheaptid) + return false; + } + + return true; +} + +/* + * colstore_attrs_used + * True if any of the Vars from this relation are required in the query + */ +static inline bool +colstore_attrs_used(PlannerInfo *root, Relids joinrelids, RelOptInfo *rel) +{ + int attroff; + ListCell *l; + AttrNumber heaptidoff = 1 - rel->min_attr; + + /* + * rel is referenced if any of it's attributes are used above the join. + * + * Note that this test only detects use of rel's attributes in higher + * join conditions and the target list. There might be such attributes in + * pushed-down conditions at this join, too. + * + * As a micro-optimization, it seems better to start with max_attr and + * count down rather than starting with min_attr and counting up, on the + * theory that the system attributes are somewhat less likely to be wanted + * and should be tested last. + */ + for (attroff = rel->max_attr - rel->min_attr; + attroff >= 0; + attroff--) + { + if (attroff == heaptidoff) + continue; + + if (!bms_is_subset(rel->attr_needed[attroff], joinrelids)) + return true; + } + + /* + * Similarly check that rel isn't needed by any PlaceHolderVars that will + * be used above the join. We only need to fail if such a PHV actually + * references some of rel's attributes; but the correct check for that is + * relatively expensive, so we first check against ph_eval_at, which must + * mention rel if the PHV uses any of-rel's attrs as non-lateral + * references. Note that if the PHV's syntactic scope is just rel, we + * can't return true even if the PHV is variable-free. + */ + foreach(l, root->placeholder_list) + { + PlaceHolderInfo *phinfo = (PlaceHolderInfo *) lfirst(l); + + if (bms_is_subset(phinfo->ph_needed, joinrelids)) + continue; /* PHV is not used above the join */ + if (bms_overlap(phinfo->ph_lateral, rel->relids)) + return true; /* it references rel laterally */ + if (!bms_overlap(phinfo->ph_eval_at, rel->relids)) + continue; /* it definitely doesn't reference rel */ + if (bms_is_subset(phinfo->ph_eval_at, rel->relids)) + return true; /* there isn't any other place to eval PHV */ + if (bms_overlap(pull_varnos((Node *) phinfo->ph_var->phexpr), + rel->relids)) + return true; /* it does reference rel */ + } + + return false; /* it does not reference rel */ +} + +/* * clause_sides_match_join * Determine whether a join clause is of the right form to use in this join. * diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 01209aa..76cac6b 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -86,6 +86,10 @@ static ForeignScan *create_foreignscan_plan(PlannerInfo *root, ForeignPath *best static CustomScan *create_customscan_plan(PlannerInfo *root, CustomPath *best_path, List *tlist, List *scan_clauses); +static ColumnStoreScan *create_colstore_scan_plan(PlannerInfo *root, + Path *best_path, + List *tlist, + List *scan_clauses); static NestLoop *create_nestloop_plan(PlannerInfo *root, NestPath *best_path, Plan *outer_plan, Plan *inner_plan); static MergeJoin *create_mergejoin_plan(PlannerInfo *root, MergePath *best_path, @@ -136,6 +140,8 @@ static CteScan *make_ctescan(List *qptlist, List *qpqual, Index scanrelid, int ctePlanId, int cteParam); static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual, Index scanrelid, int wtParam); +static ColumnStoreScan *make_colstore_scan(List *targetlist, List *qpqual, + Index scanrelid); static BitmapAnd *make_bitmap_and(List *bitmapplans); static BitmapOr *make_bitmap_or(List *bitmapplans); static NestLoop *make_nestloop(List *tlist, @@ -249,6 +255,7 @@ create_plan_recurse(PlannerInfo *root, Path *best_path) case T_WorkTableScan: case T_ForeignScan: case T_CustomScan: + case T_ColumnStoreScan: plan = create_scan_plan(root, best_path); break; case T_HashJoin: @@ -443,6 +450,13 @@ create_scan_plan(PlannerInfo *root, Path *best_path) scan_clauses); break; + case T_ColumnStoreScan: + plan = (Plan *) create_colstore_scan_plan(root, + best_path, + tlist, + scan_clauses); + break; + default: elog(ERROR, "unrecognized node type: %d", (int) best_path->pathtype); @@ -2271,6 +2285,43 @@ create_customscan_plan(PlannerInfo *root, CustomPath *best_path, return cplan; } +/* + * create_colstore_scan_plan + * Create a ColumnStoreScan plan for 'best_path'. + * + * Returns a Plan node. + */ +static ColumnStoreScan * +create_colstore_scan_plan(PlannerInfo *root, Path *best_path, List *tlist, + List *scan_clauses) +{ + ColumnStoreScan *scan_plan; + Index scan_relid = best_path->parent->relid; + + /* it should be a base rel... */ + Assert(scan_relid > 0); + Assert(best_path->parent->rtekind == RTE_RELATION); + + /* Sort clauses into best execution order */ + scan_clauses = order_qual_clauses(root, scan_clauses); + + /* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */ + scan_clauses = extract_actual_clauses(scan_clauses, false); + + /* Replace any outer-relation variables with nestloop params */ + if (best_path->param_info) + { + scan_clauses = (List *) + replace_nestloop_params(root, (Node *) scan_clauses); + } + + scan_plan = make_colstore_scan(tlist, scan_clauses, scan_relid); + + copy_generic_path_info(&scan_plan->scan.plan, (Path *) best_path); + + return scan_plan; +} + /***************************************************************************** * @@ -3782,6 +3833,22 @@ make_foreignscan(List *qptlist, return node; } +static ColumnStoreScan * +make_colstore_scan(List *targetlist, List *qpqual, Index scanrelid) +{ + ColumnStoreScan *node = makeNode(ColumnStoreScan); + Plan *plan = &node->scan.plan; + + /* cost should be inserted by caller */ + plan->targetlist = targetlist; + plan->qual = qpqual; + plan->lefttree = NULL; + plan->righttree = NULL; + node->scan.scanrelid = scanrelid; + + return node; +} + Append * make_append(List *appendplans, List *tlist) { diff --git a/src/backend/optimizer/plan/initsplan.c b/src/backend/optimizer/plan/initsplan.c index f2dce6a..5dbfa6d 100644 --- a/src/backend/optimizer/plan/initsplan.c +++ b/src/backend/optimizer/plan/initsplan.c @@ -14,7 +14,11 @@ */ #include "postgres.h" +#include "access/sysattr.h" +#include "catalog/pg_class.h" +#include "catalog/pg_operator.h" #include "catalog/pg_type.h" +#include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "optimizer/clauses.h" #include "optimizer/cost.h" @@ -22,12 +26,14 @@ #include "optimizer/pathnode.h" #include "optimizer/paths.h" #include "optimizer/placeholder.h" +#include "optimizer/plancat.h" #include "optimizer/planmain.h" #include "optimizer/planner.h" #include "optimizer/prep.h" #include "optimizer/restrictinfo.h" #include "optimizer/var.h" #include "parser/analyze.h" +#include "parser/parsetree.h" #include "rewrite/rewriteManip.h" #include "utils/lsyscache.h" @@ -44,7 +50,34 @@ typedef struct PostponedQual Relids relids; /* the set of baserels it references */ } PostponedQual; +/* + * cstore_replace_vars_context (and its subsidiary AttributeMap) are used for + * replace_rte_variables in expand_column_store_relations; they keep track of + * column stores for each individual table while looking to join the column + * stores. + */ +typedef struct AttributeMap +{ + int colstoreidx; /* Index of colstore list */ + AttrNumber attrno; /* Varattno in colstore */ +} AttributeMap; +typedef struct cstore_replace_vars_context +{ + int mapsize; /* number of elements in attrmap */ + AttributeMap *attrmap; /* map of where Vars are physically located */ + Bitmapset **selectedCols; /* bitmap of found columns, or NULL if none */ + int *colstore_varno; /* array of varnos reserved for colstore rti, + * or zero if not reserved */ + int nextvarno; /* next free varno to use */ + List *colstores; +} cstore_replace_vars_context; + +static Node *expand_rel_to_join(PlannerInfo *root, RangeTblRef *rtr); +static Node *replace_csvars_callback(Var *var, + replace_rte_variables_context *context); +static OpExpr *make_columnar_joinqual(PlannerInfo *root, RangeTblRef *heapref, + RangeTblRef *cstoreref); static void extract_lateral_references(PlannerInfo *root, RelOptInfo *brel, Index rtindex); static List *deconstruct_recurse(PlannerInfo *root, Node *jtnode, @@ -127,6 +160,321 @@ add_base_rels_to_query(PlannerInfo *root, Node *jtnode) (int) nodeTag(jtnode)); } +/* + * expand_column_store_relations + * Scan the query's jointree, and apply expand_rel_to_join to all RangeTbl + * references therein. + */ +Node * +expand_column_store_relations(PlannerInfo *root, Node *jtnode) +{ + if (jtnode == NULL) + return NULL; + if (IsA(jtnode, RangeTblRef)) + { + RangeTblRef *rtr = (RangeTblRef *) jtnode; + + return expand_rel_to_join(root, rtr); + } + else if (IsA(jtnode, FromExpr)) + { + FromExpr *f = (FromExpr *) jtnode; + ListCell *l; + + foreach(l, f->fromlist) + lfirst(l) = expand_column_store_relations(root, (Node *) lfirst(l)); + } + else if (IsA(jtnode, JoinExpr)) + { + JoinExpr *j = (JoinExpr *) jtnode; + + j->larg = expand_column_store_relations(root, j->larg); + j->rarg = expand_column_store_relations(root, j->rarg); + } + else + elog(ERROR, "unrecognized node type: %d", + (int) nodeTag(jtnode)); + + return jtnode; +} + +/***************************************************************************** + * + * COLUMN STORES + * + *****************************************************************************/ + +/* + * expand_rel_to_join + * Expand a table RTE considering possible column stores + * + * Given a RangeTblRef, see whether it references a table containing column + * stores, and if so, add additional relations for the column stores used + * anywhere in the query, and further relations for the joins between those + * new relations and the original table relation. + * + * Additionally, the entire tree is scanned for Vars that reference the + * original relation; those that point to columns that are actually in the + * column store are mutated into referencing the newly added RTE instead. + */ +static Node * +expand_rel_to_join(PlannerInfo *root, RangeTblRef *rtr) +{ + RangeTblEntry *rte = rt_fetch(rtr->rtindex, root->parse->rtable); + Node *parentnode = (Node *) rtr; + + if (rte->rtekind == RTE_RELATION && rte->relhascstore) + { + RelOptInfo *rel = makeNode(RelOptInfo); + + /* + * XXX It's a shame we need to call get_relation_info() now, and again + * in add_base_rels_to_query(). Maybe we can do some caching to get + * around the duplicate calls + */ + get_relation_info(root, rte->relid, false, rel); + + if (rel->cstlist != NIL) + { + ListCell *l; + cstore_replace_vars_context context; + int colstoreidx; + int ncstores = list_length(rel->cstlist); + + context.mapsize = rel->max_attr + 1; + context.attrmap = (AttributeMap *) + palloc0(sizeof(AttributeMap) * context.mapsize); + + context.selectedCols = + (Bitmapset **) palloc(sizeof(Bitmapset *) * ncstores); + + context.colstore_varno = (int *) palloc0(sizeof(int) * ncstores); + context.nextvarno = list_length(root->parse->rtable) + 1; + context.colstores = NIL; + + /* + * Here we build an array which marks the physical location of each + * Var which logically belongs to this relation. Vars can be + * stored in the heap, or any of the relation's column stores. We + * index the array by the columns logical location within the + * relation, this is the same as the varattno before it is + * modified later. + * + * Our array elements are ColumnStoreAttrMaps, which contains the + * column store number, as the position in cstlist, and also the + * varattno number of where the attribute is located in the column + * store. Later, when processing this array, we're able to + * determine Vars which belong to the heap by the 'attrno' of these + * being set to 0. + */ + + colstoreidx = 0; + foreach(l, rel->cstlist) + { + ColumnStoreOptInfo *info = (ColumnStoreOptInfo *) lfirst(l); + int col; + + for (col = 0; col < info->ncolumns; col++) + { + AttributeMap *mapitem; + + /* XXX protect against out of range values ? */ + mapitem = &context.attrmap[info->cstkeys[col]]; + + /* + * The actual attribute number of the attribute in the + * colstore is the index of cstkey's + 2. The reason for + * +2 here is that we need to account for varattnos being + * 1-based, and the cstkeys[] array being zero 0-based. + * Also the first user defined varattno in the colstore is + * 2, since varattno 1 is used to store the heap's ctid. + */ + + mapitem->attrno = col + 2; + mapitem->colstoreidx = colstoreidx; + } + + context.selectedCols[colstoreidx] = NULL; + colstoreidx++; + } + + /* + * Traverse the parse tree to mutate all Vars which belong to this + * relation, updating the colstores list in the context struct. + */ + replace_rte_variables((Node *) root->parse, rtr->rtindex, 0, + replace_csvars_callback, + (void *) &context, NULL); + + /* + * Add an inner join relation to the rangetable for each necessary + * column store. Note that we must do this in the same order that we + * found the Vars, as the varnos were "reserved" in that order. + */ + foreach(l, context.colstores) + { + ColumnStoreOptInfo *info; + JoinExpr *join; + RangeTblEntry *joinrte = makeNode(RangeTblEntry); + RangeTblEntry *newrte = makeNode(RangeTblEntry); + RangeTblRef *newrtr = makeNode(RangeTblRef); + ColstoreRelInfo *cstinfo; + int newrelid; + int newjoinid; + + colstoreidx = lfirst_int(l); + + /* + * lookup the column store by index. + */ + info = (ColumnStoreOptInfo *) list_nth(rel->cstlist, colstoreidx); + + newrelid = context.colstore_varno[colstoreidx]; + newjoinid = newrelid - 1; + + /* + * We've found some Vars which belong to this colstore, + * so add a new join. + */ + joinrte->rtekind = RTE_JOIN; + joinrte->relid = InvalidOid; + joinrte->subquery = NULL; + joinrte->jointype = JOIN_INNER; + joinrte->lateral = false; + joinrte->inh = false; /* never true for joins */ + joinrte->inFromCl = false; + + joinrte->requiredPerms = 0; + joinrte->checkAsUser = InvalidOid; + joinrte->selectedCols = NULL; + joinrte->insertedCols = NULL; + joinrte->updatedCols = NULL; + joinrte->eref = makeAlias("unnamed_join", NIL); + + newrte->relkind = RTE_RELATION; + newrte->relid = info->colstoreoid; + newrte->relkind = RELKIND_COLUMN_STORE; + newrte->subquery = NULL; + newrte->security_barrier = false; + newrte->eref = makeNode(Alias); + newrte->eref->aliasname = get_cstore_name(info->colstoreoid); + newrte->selectedCols = context.selectedCols[colstoreidx]; + + newrtr->rtindex = newrelid; + + join = makeNode(JoinExpr); + join->jointype = JOIN_INNER; + join->isNatural = false; + join->larg = (Node *) parentnode; + join->rarg = (Node *) newrtr; + join->usingClause = NIL; + join->quals = (Node *) list_make1(make_columnar_joinqual(root, rtr, newrtr)); + join->alias = NULL; + join->rtindex = newjoinid; + + /* + * Carefully add the 2 new RTEs. The join comes before the + * column store RTE, as this is the order in which we assigned + * them above. + */ + root->parse->rtable = lappend(root->parse->rtable, joinrte); + root->parse->rtable = lappend(root->parse->rtable, newrte); + parentnode = (Node *) join; + + /* + * cons up a new ColstoreRelInfo for this colstore and add it + * to the root list + */ + cstinfo = makeNode(ColstoreRelInfo); + cstinfo->parent_relid = rtr->rtindex; + cstinfo->child_relid = newrelid; + cstinfo->child_oid = info->colstoreoid; + root->colstore_rel_list = lappend(root->colstore_rel_list, cstinfo); + } + + pfree(context.attrmap); + pfree(context.selectedCols); + list_free(context.colstores); + pfree(rel); + } + } + + return parentnode; +} + +static Node * +replace_csvars_callback(Var *var, replace_rte_variables_context *context) +{ + cstore_replace_vars_context *rcon; + AttributeMap *map; + + rcon = (cstore_replace_vars_context *) context->callback_arg; + map = &rcon->attrmap[var->varattno]; + + /* + * Modify this Var if the map indicates it's not in the heap. Also, for + * Vars that belong to colstores that we hadn't previously seen, reserve + * varnos for both the column store relation and the join relation. + */ + if (map->attrno != 0) + { + int colstoreidx = map->colstoreidx; + + /* + * Determine if we've seen any Vars belonging to this column store + * before. If we have then we've already assigned a varno for the new + * column store relation. If we've not, then we'll need to allocate a + * varno, both for the join and for the new column store. + */ + if (rcon->colstore_varno[colstoreidx] == 0) + { + /* + * We'll give this colstore nextvarno + 1. The join rte will be + * nextvarno. + */ + rcon->colstore_varno[colstoreidx] = rcon->nextvarno + 1; + rcon->nextvarno += 2; + + /* + * We must remember the order which we assigned the varnos to the + * column stores so we can add new rtable items later in the same + * order that we assigned the varnos. + */ + rcon->colstores = lappend_int(rcon->colstores, colstoreidx); + } + + /* Modify the Var to reference the column store */ + var->varoattno = var->varattno = map->attrno; + var->varnoold = var->varno = rcon->colstore_varno[colstoreidx]; + rcon->selectedCols[colstoreidx] = + bms_add_member(rcon->selectedCols[colstoreidx], map->attrno); + } + + return (Node *) var; +} + +/* + * Return an expression that represents the condition that joins a column store + * with its owning table. + */ +static OpExpr * +make_columnar_joinqual(PlannerInfo *root, RangeTblRef *heapref, + RangeTblRef *cstoreref) +{ + OpExpr *joinqual = makeNode(OpExpr); + Var *heapctid; + Var *cstorekey; + + heapctid = makeVar(heapref->rtindex, SelfItemPointerAttributeNumber, + TIDOID, -1, InvalidOid, 0); + cstorekey = makeVar(cstoreref->rtindex, 1, TIDOID, -1, InvalidOid, 0); + + joinqual->args = list_make2(heapctid, cstorekey); + joinqual->opno = TIDEqualOperator; + + return joinqual; +} + /***************************************************************************** * diff --git a/src/backend/optimizer/plan/planmain.c b/src/backend/optimizer/plan/planmain.c index 894f968..5e075f9 100644 --- a/src/backend/optimizer/plan/planmain.c +++ b/src/backend/optimizer/plan/planmain.c @@ -102,8 +102,9 @@ query_planner(PlannerInfo *root, List *tlist, /* * Init planner lists to empty. * - * NOTE: append_rel_list was set up by subquery_planner, so do not touch - * here; eq_classes and minmax_aggs may contain data already, too. + * NOTE: append_rel_list and colstore_rel_list were set up by + * subquery_planner, so do not touch here; eq_classes and minmax_aggs may + * contain data already, too. */ root->join_rel_list = NIL; root->join_rel_hash = NULL; @@ -118,6 +119,14 @@ query_planner(PlannerInfo *root, List *tlist, root->initial_rels = NIL; /* + * For any Relations which have Vars stored in column stores we must + * transform the jointree to include a join to the required colstore. + */ + parse->jointree = + (FromExpr *) expand_column_store_relations(root, + (Node *) parse->jointree); + + /* * Make a flattened version of the rangetable for faster access (this is * OK because the rangetable won't change any more), and set up an empty * array for indexing base relations. @@ -126,14 +135,15 @@ query_planner(PlannerInfo *root, List *tlist, /* * Construct RelOptInfo nodes for all base relations in query, and - * indirectly for all appendrel member relations ("other rels"). This - * will give us a RelOptInfo for every "simple" (non-join) rel involved in - * the query. + * indirectly for all appendrel member relations and column store rels + * ("other rels"). This will give us a RelOptInfo for every "simple" + * (non-join) rel involved in the query. * * Note: the reason we find the rels by searching the jointree and - * appendrel list, rather than just scanning the rangetable, is that the - * rangetable may contain RTEs for rels not actively part of the query, - * for example views. We don't want to make RelOptInfos for them. + * appendrel and colstore lists, rather than just scanning the rangetable, + * is that the rangetable may contain RTEs for rels not actively part of + * the query, for example views. We don't want to make RelOptInfos for + * them. */ add_base_rels_to_query(root, (Node *) parse->jointree); diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index 2c04f5c..9ef7c45 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -409,6 +409,7 @@ subquery_planner(PlannerGlobal *glob, Query *parse, root->multiexpr_params = NIL; root->eq_classes = NIL; root->append_rel_list = NIL; + root->colstore_rel_list = NIL; root->rowMarks = NIL; root->hasInheritedTarget = false; root->grouping_map = NULL; @@ -4642,6 +4643,7 @@ plan_cluster_use_sort(Oid tableOid, Oid indexOid) rte->rtekind = RTE_RELATION; rte->relid = tableOid; rte->relkind = RELKIND_RELATION; /* Don't be too picky. */ + rte->relhascstore = false; /* XXX does this matter? */ rte->lateral = false; rte->inh = false; rte->inFromCl = true; diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 12e9290..f72e307 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -595,7 +595,17 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset) case T_CustomScan: set_customscan_references(root, (CustomScan *) plan, rtoffset); break; + case T_ColumnStoreScan: + { + ColumnStoreScan *splan = (ColumnStoreScan *) plan; + splan->scan.scanrelid += rtoffset; + splan->scan.plan.targetlist = + fix_scan_list(root, splan->scan.plan.targetlist, rtoffset); + splan->scan.plan.qual = + fix_scan_list(root, splan->scan.plan.qual, rtoffset); + } + break; case T_NestLoop: case T_MergeJoin: case T_HashJoin: diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c index 82414d4..20d5781 100644 --- a/src/backend/optimizer/plan/subselect.c +++ b/src/backend/optimizer/plan/subselect.c @@ -2432,6 +2432,10 @@ finalize_plan(PlannerInfo *root, Plan *plan, Bitmapset *valid_params, } break; + case T_ColumnStoreScan: + /* TODO what to do here? */ + break; + case T_ModifyTable: { ModifyTable *mtplan = (ModifyTable *) plan; diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index ec0910d..c9e0195 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -1529,6 +1529,26 @@ create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel, } /* + * create_colstore_scan_path + * Creates a path corresponding to a column store scan, + * returning the pathnode. + */ +Path * +create_colstore_scan_path(PlannerInfo *root, RelOptInfo *rel) +{ + Path *pathnode = makeNode(Path); + + pathnode->pathtype = T_ColumnStoreScan; + pathnode->parent = rel; + pathnode->param_info = NULL; + pathnode->pathkeys = NIL; /* colstore scan has unordered results */ + + cost_colstore_scan(pathnode, root, rel, pathnode->param_info); + + return pathnode; +} + +/* * calc_nestloop_required_outer * Compute the required_outer set for a nestloop join path * diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c index 9442e5f..415f52f 100644 --- a/src/backend/optimizer/util/plancat.c +++ b/src/backend/optimizer/util/plancat.c @@ -27,6 +27,7 @@ #include "catalog/catalog.h" #include "catalog/dependency.h" #include "catalog/heap.h" +#include "catalog/pg_cstore.h" #include "foreign/fdwapi.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -71,6 +72,7 @@ static List *build_index_tlist(PlannerInfo *root, IndexOptInfo *index, * min_attr lowest valid AttrNumber * max_attr highest valid AttrNumber * indexlist list of IndexOptInfos for relation's indexes + * cstlist list of ColumnStoreOptInfos for relation's colstores * serverid if it's a foreign table, the server OID * fdwroutine if it's a foreign table, the FDW function pointers * pages number of pages @@ -93,6 +95,7 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, Relation relation; bool hasindex; List *indexinfos = NIL; + List *colstoreinfos = NIL; /* * We need not lock the relation since it was already locked, either by @@ -381,6 +384,91 @@ get_relation_info(PlannerInfo *root, Oid relationObjectId, bool inhparent, rel->indexlist = indexinfos; + /* Grab column store info using the relcache, while we have it */ + if (relation->rd_rel->relhascstore) + { + List *colstoreoidlist; + ListCell *l; + LOCKMODE lmode; + + colstoreoidlist = RelationGetColStoreList(relation); + + /* + * For each column store, we get the same type of lock that the + * executor will need, and do not release it. This saves a couple + * of trips to the shared lock manager while not creating any real + * loss of concurrency, because no schema changes could be + * happening on the colstore while we hold lock on the parent rel, + * and neither lock type blocks any other kind of colstore operation. + */ + if (rel->relid == root->parse->resultRelation) + lmode = RowExclusiveLock; + else + lmode = AccessShareLock; + + foreach(l, colstoreoidlist) + { + Oid colstoreoid = lfirst_oid(l); + Relation colstoreRelation; + Form_pg_cstore colstore; + ColumnStoreOptInfo *info; + int ncolumns; + int i; + + /* + * Extract info from the relation descriptor for the colstore. + * + * FIXME There's no 'rd_cstore' in RelationData at the moment, + * so it needs to be added and integrated into relcache + * (just a bit of copy'n'paste programming using the + * rd_index logic). + * + * TODO Define colstore_open(), similar to index_open(). + */ + colstoreRelation = relation_open(colstoreoid, lmode); + colstore = colstoreRelation->rd_cstore; + + /* XXX Invalid and not-yet-usable colstores would be handled here. */ + + info = makeNode(ColumnStoreOptInfo); + + info->colstoreoid = colstore->cststoreid; + info->reltablespace = + RelationGetForm(colstoreRelation)->reltablespace; + info->rel = rel; + info->ncolumns = ncolumns = colstore->cstnatts; + info->cstkeys = (int *) palloc(sizeof(int) * ncolumns); + + for (i = 0; i < ncolumns; i++) + info->cstkeys[i] = colstore->cstatts.values[i]; + + /* + * TODO This is where to fetch AM for the colstore (see how + * the index are handled above. + */ + + /* + * XXX this is where indexes build targetlists. Colstores don't + * have these (at least not ATM). + */ + + /* + * Estimate the colstore size. We don't support partial + * colstores, we just use the same reltuples as the parent. + */ + info->pages = RelationGetNumberOfBlocks(colstoreRelation); + info->tuples = rel->tuples; + + relation_close(colstoreRelation, NoLock); + + colstoreinfos = lcons(info, colstoreinfos); + } + + list_free(colstoreoidlist); + } + + rel->cstlist = colstoreinfos; + /* Grab foreign-table info using the relcache, while we have it */ if (relation->rd_rel->relkind == RELKIND_FOREIGN_TABLE) { @@ -917,6 +1005,7 @@ estimate_rel_size(Relation rel, int32 *attr_widths, *tuples = 1; *allvisfrac = 0; break; + case RELKIND_COLUMN_STORE: case RELKIND_FOREIGN_TABLE: /* Just use whatever's in pg_class */ *pages = rel->rd_rel->relpages; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 223ef17..ae7259a 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -171,6 +171,7 @@ static TypeName *TableFuncTypeName(List *columns); static RangeVar *makeRangeVarFromAnyName(List *names, int position, core_yyscan_t yyscanner); static void SplitColQualList(List *qualList, List **constraintList, CollateClause **collClause, + ColumnStoreClause **cstoreClause, core_yyscan_t yyscanner); static void processCASbits(int cas_bits, int location, const char *constrType, bool *deferrable, bool *initdeferred, bool *not_valid, @@ -263,7 +264,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); DeallocateStmt PrepareStmt ExecuteStmt DropOwnedStmt ReassignOwnedStmt AlterTSConfigurationStmt AlterTSDictionaryStmt - CreateMatViewStmt RefreshMatViewStmt + CreateMatViewStmt RefreshMatViewStmt CreateColumnStoreAMStmt %type select_no_parens select_with_parens select_clause simple_select values_clause @@ -378,6 +379,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type opt_fdw_options fdw_options %type fdw_option +%type opt_cstam_options cstam_options +%type cstam_option + %type OptTempTableName %type into_clause create_as_target create_mv_target @@ -501,7 +505,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type unreserved_keyword type_func_name_keyword %type col_name_keyword reserved_keyword -%type TableConstraint TableLikeClause +%type TableConstraint TableLikeClause ColStoreClause %type TableLikeOptionList TableLikeOption %type ColQualList %type ColConstraint ColConstraintElem ConstraintAttr @@ -604,7 +608,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOGGED - MAPPING MATCH MATERIALIZED MAXVALUE MINUTE_P MINVALUE MODE MONTH_P MOVE + MAPPING MATCH MATERIALIZED MAXVALUE METHOD MINUTE_P MINVALUE MODE MONTH_P MOVE NAME_P NAMES NATIONAL NATURAL NCHAR NEXT NO NONE NOT NOTHING NOTIFY NOTNULL NOWAIT NULL_P NULLIF @@ -627,7 +631,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); SAVEPOINT SCHEMA SCROLL SEARCH SECOND_P SECURITY SELECT SEQUENCE SEQUENCES SERIALIZABLE SERVER SESSION SESSION_USER SET SETS SETOF SHARE SHOW SIMILAR SIMPLE SKIP SMALLINT SNAPSHOT SOME SQL_P STABLE STANDALONE_P START - STATEMENT STATISTICS STDIN STDOUT STORAGE STRICT_P STRIP_P SUBSTRING + STATEMENT STATISTICS STDIN STDOUT STORAGE STORE STRICT_P STRIP_P SUBSTRING SYMMETRIC SYSID SYSTEM_P TABLE TABLES TABLESAMPLE TABLESPACE TEMP TEMPLATE TEMPORARY TEXT_P THEN @@ -792,6 +796,7 @@ stmt : | CreateAsStmt | CreateAssertStmt | CreateCastStmt + | CreateColumnStoreAMStmt | CreateConversionStmt | CreateDomainStmt | CreateExtensionStmt @@ -2941,11 +2946,13 @@ TableElement: columnDef { $$ = $1; } | TableLikeClause { $$ = $1; } | TableConstraint { $$ = $1; } + | ColStoreClause { $$ = $1; } ; TypedTableElement: columnOptions { $$ = $1; } | TableConstraint { $$ = $1; } + | ColStoreClause { $$ = $1; } ; columnDef: ColId Typename create_generic_options ColQualList @@ -2963,6 +2970,7 @@ columnDef: ColId Typename create_generic_options ColQualList n->collOid = InvalidOid; n->fdwoptions = $3; SplitColQualList($4, &n->constraints, &n->collClause, + &n->cstoreClause, yyscanner); n->location = @1; $$ = (Node *)n; @@ -2983,6 +2991,7 @@ columnOptions: ColId WITH OPTIONS ColQualList n->cooked_default = NULL; n->collOid = InvalidOid; SplitColQualList($4, &n->constraints, &n->collClause, + &n->cstoreClause, yyscanner); n->location = @1; $$ = (Node *)n; @@ -3018,6 +3027,20 @@ ColConstraint: n->location = @1; $$ = (Node *) n; } + | COLUMN STORE ColId USING ColId opt_reloptions OptTableSpace + { + /* + * Note: as with COLLATE, this is here only temporarily. + */ + ColumnStoreClause *n = makeNode(ColumnStoreClause); + n->name = $3; + n->storetype = $5; + n->columns = NIL; + n->options = $6; + n->tablespacename = $7; + n->location = @1; + $$ = (Node *) n; + } ; /* DEFAULT NULL is already the default for Postgres. @@ -3199,6 +3222,19 @@ TableConstraint: | ConstraintElem { $$ = $1; } ; +ColStoreClause: + COLUMN STORE ColId USING ColId '(' columnList ')' opt_reloptions OptTableSpace + { + ColumnStoreClause *n = makeNode(ColumnStoreClause); + n->name = $3; + n->storetype = $5; + n->columns = $7; + n->options = $9; + n->tablespacename = $10; + $$ = (Node *) n; + } + ; + ConstraintElem: CHECK '(' a_expr ')' ConstraintAttributeSpec { @@ -4193,6 +4229,38 @@ opt_fdw_options: | /*EMPTY*/ { $$ = NIL; } ; + +/***************************************************************************** + * + * QUERY: + * CREATE COLUMN STORE ACCESS METHOD name options + * + *****************************************************************************/ + +CreateColumnStoreAMStmt: CREATE COLUMN STORE ACCESS METHOD name opt_cstam_options + { + CreateColumnStoreAMStmt *n = makeNode(CreateColumnStoreAMStmt); + n->cstamname = $6; + n->func_options = $7; + $$ = (Node *) n; + } + ; + +cstam_option: + HANDLER handler_name { $$ = makeDefElem("handler", (Node *)$2); } + | NO HANDLER { $$ = makeDefElem("handler", NULL); } + ; + +cstam_options: + cstam_option { $$ = list_make1($1); } + | cstam_options cstam_option { $$ = lappend($1, $2); } + ; + +opt_cstam_options: + cstam_options { $$ = $1; } + | /*EMPTY*/ { $$ = NIL; } + ; + /***************************************************************************** * * QUERY : @@ -8989,6 +9057,7 @@ CreateDomainStmt: n->domainname = $3; n->typeName = $5; SplitColQualList($6, &n->constraints, &n->collClause, + NULL, yyscanner); $$ = (Node *)n; } @@ -13778,6 +13847,7 @@ unreserved_keyword: | MATCH | MATERIALIZED | MAXVALUE + | METHOD | MINUTE_P | MINVALUE | MODE @@ -13871,6 +13941,7 @@ unreserved_keyword: | STDIN | STDOUT | STORAGE + | STORE | STRICT_P | STRIP_P | SYSID @@ -14731,6 +14802,7 @@ makeRangeVarFromAnyName(List *names, int position, core_yyscan_t yyscanner) static void SplitColQualList(List *qualList, List **constraintList, CollateClause **collClause, + ColumnStoreClause **cstoreClause, core_yyscan_t yyscanner) { ListCell *cell; @@ -14761,6 +14833,22 @@ SplitColQualList(List *qualList, parser_errposition(c->location))); *collClause = c; } + else if (IsA(n, ColumnStoreClause)) + { + ColumnStoreClause *c = (ColumnStoreClause *) n; + + if (cstoreClause == NULL) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("COLUMN STORE clause not allowed here"), + parser_errposition(c->location))); + if (*cstoreClause) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("multiple COLUMN STORE clauses not allowed"), + parser_errposition(c->location))); + *cstoreClause = c; + } else elog(ERROR, "unexpected node type %d", (int) n->type); /* remove non-Constraint nodes from qualList */ diff --git a/src/backend/parser/parse_relation.c b/src/backend/parser/parse_relation.c index 6f569da..dea40f5 100644 --- a/src/backend/parser/parse_relation.c +++ b/src/backend/parser/parse_relation.c @@ -1193,6 +1193,7 @@ addRangeTableEntry(ParseState *pstate, rel = parserOpenTable(pstate, relation, lockmode); rte->relid = RelationGetRelid(rel); rte->relkind = rel->rd_rel->relkind; + rte->relhascstore = rel->rd_rel->relhascstore; /* * Build the list of effective column names using user-supplied aliases @@ -1255,6 +1256,7 @@ addRangeTableEntryForRelation(ParseState *pstate, rte->alias = alias; rte->relid = RelationGetRelid(rel); rte->relkind = rel->rd_rel->relkind; + rte->relhascstore = rel->rd_rel->relhascstore; /* * Build the list of effective column names using user-supplied aliases diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 4c24c13..18ae2cd 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -274,6 +274,10 @@ transformCreateStmt(CreateStmt *stmt, const char *queryString) /* process later */ break; + case T_ColumnStoreClause: + stmt->colstores = lappend(stmt->colstores, element); + break; + default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(element)); diff --git a/src/backend/rewrite/rewriteDefine.c b/src/backend/rewrite/rewriteDefine.c index 39c83a6..3c5dd5e 100644 --- a/src/backend/rewrite/rewriteDefine.c +++ b/src/backend/rewrite/rewriteDefine.c @@ -447,6 +447,12 @@ DefineQueryRewrite(char *rulename, errmsg("could not convert table \"%s\" to a view because it has indexes", RelationGetRelationName(event_relation)))); + if (event_relation->rd_rel->relhascstore) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("could not convert table \"%s\" to a view because it has column stores", + RelationGetRelationName(event_relation)))); + if (event_relation->rd_rel->relhassubclass) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), @@ -606,6 +612,7 @@ DefineQueryRewrite(char *rulename, classForm->relallvisible = 0; classForm->reltoastrelid = InvalidOid; classForm->relhasindex = false; + classForm->relhascstore = false; classForm->relkind = RELKIND_VIEW; classForm->relhasoids = false; classForm->relhaspkey = false; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index e81bbc6..711c64f 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -209,6 +209,7 @@ check_xact_readonly(Node *parsetree) case T_CreateForeignTableStmt: case T_ImportForeignSchemaStmt: case T_SecLabelStmt: + case T_CreateColumnStoreAMStmt: PreventCommandIfReadOnly(CreateCommandTag(parsetree)); PreventCommandIfParallelMode(CreateCommandTag(parsetree)); break; @@ -1520,6 +1521,10 @@ ProcessUtilitySlow(Node *parsetree, address = ExecSecLabelStmt((SecLabelStmt *) parsetree); break; + case T_CreateColumnStoreAMStmt: + CreateColumnStoreAM((CreateColumnStoreAMStmt *) parsetree); + break; + default: elog(ERROR, "unrecognized node type: %d", (int) nodeTag(parsetree)); @@ -2665,6 +2670,10 @@ CreateCommandTag(Node *parsetree) } break; + case T_CreateColumnStoreAMStmt: + tag = "CREATE COLUMN STORE ACCESS METHOD"; + break; + default: elog(WARNING, "unrecognized node type: %d", (int) nodeTag(parsetree)); diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c index 5ee59d0..a432832 100644 --- a/src/backend/utils/adt/dbsize.c +++ b/src/backend/utils/adt/dbsize.c @@ -445,6 +445,46 @@ calculate_indexes_size(Relation rel) return size; } +/* + * Calculate total on-disk size of all column stores attached to the given table. + * + * Can be applied safely to a column store, but you'll just get zero. + */ +static int64 +calculate_column_stores_size(Relation rel) +{ + int64 size = 0; + + /* + * Aggregate all indexes on the given relation + */ + if (rel->rd_rel->relhascstore) + { + List *cstore_oids = RelationGetColStoreList(rel); + ListCell *cell; + + foreach(cell, cstore_oids) + { + Oid cstoreOid = lfirst_oid(cell); + Relation cstoreRel; + ForkNumber forkNum; + + cstoreRel = relation_open(cstoreOid, AccessShareLock); + + for (forkNum = 0; forkNum <= MAX_FORKNUM; forkNum++) + size += calculate_relation_size(&(cstoreRel->rd_node), + cstoreRel->rd_backend, + forkNum); + + relation_close(cstoreRel, AccessShareLock); + } + + list_free(cstore_oids); + } + + return size; +} + Datum pg_table_size(PG_FUNCTION_ARGS) { @@ -483,6 +523,25 @@ pg_indexes_size(PG_FUNCTION_ARGS) PG_RETURN_INT64(size); } +Datum +pg_column_stores_size(PG_FUNCTION_ARGS) +{ + Oid relOid = PG_GETARG_OID(0); + Relation rel; + int64 size; + + rel = try_relation_open(relOid, AccessShareLock); + + if (rel == NULL) + PG_RETURN_NULL(); + + size = calculate_column_stores_size(rel); + + relation_close(rel, AccessShareLock); + + PG_RETURN_INT64(size); +} + /* * Compute the on-disk size of all files for the relation, * including heap data, index data, toast data, FSM, VM. @@ -503,6 +562,11 @@ calculate_total_relation_size(Relation rel) */ size += calculate_indexes_size(rel); + /* + * Add size of all attached column stores as well + */ + size += calculate_column_stores_size(rel); + return size; } diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c index 5b809aa..7680777 100644 --- a/src/backend/utils/adt/pseudotypes.c +++ b/src/backend/utils/adt/pseudotypes.c @@ -401,6 +401,33 @@ tsm_handler_out(PG_FUNCTION_ARGS) /* + * cstore_handler_in - input routine for pseudo-type CSTORE_HANDLER. + */ +Datum +cstore_handler_in(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot accept a value of type cstore_handler"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + +/* + * cstore_handler_out - output routine for pseudo-type CSTORE_HANDLER. + */ +Datum +cstore_handler_out(PG_FUNCTION_ARGS) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot display a value of type cstore_handler"))); + + PG_RETURN_VOID(); /* keep compiler quiet */ +} + + +/* * internal_in - input routine for pseudo-type INTERNAL. */ Datum diff --git a/src/backend/utils/adt/ri_triggers.c b/src/backend/utils/adt/ri_triggers.c index 6569fdb..86c81cc 100644 --- a/src/backend/utils/adt/ri_triggers.c +++ b/src/backend/utils/adt/ri_triggers.c @@ -2282,6 +2282,9 @@ RI_Initial_Check(Trigger *trigger, Relation fk_rel, Relation pk_rel) * works because it changes user IDs on the fly.) * * XXX are there any other show-stopper conditions to check? + * + * XXX we don't care about ->relhascstore because these RTEs are only + * used for permissions checking. */ pkrte = makeNode(RangeTblEntry); pkrte->rtekind = RTE_RELATION; diff --git a/src/backend/utils/adt/ruleutils.c b/src/backend/utils/adt/ruleutils.c index 280808a..fbec703 100644 --- a/src/backend/utils/adt/ruleutils.c +++ b/src/backend/utils/adt/ruleutils.c @@ -859,7 +859,12 @@ pg_get_triggerdef_worker(Oid trigid, bool pretty) relkind = get_rel_relkind(trigrec->tgrelid); - /* Build minimal OLD and NEW RTEs for the rel */ + /* + * Build minimal OLD and NEW RTEs for the rel. + * + * XXX we don't care about relhascstore because these RTEs are only + * used for rule expansion + */ oldrte = makeNode(RangeTblEntry); oldrte->rtekind = RTE_RELATION; oldrte->relid = trigrec->tgrelid; @@ -2540,6 +2545,7 @@ deparse_context_for(const char *aliasname, Oid relid) rte->rtekind = RTE_RELATION; rte->relid = relid; rte->relkind = RELKIND_RELATION; /* no need for exactness here */ + rte->relhascstore = false; /* nor here */ rte->alias = makeAlias(aliasname, NIL); rte->eref = rte->alias; rte->lateral = false; diff --git a/src/backend/utils/cache/catcache.c b/src/backend/utils/cache/catcache.c index 577c059..2b7fb60 100644 --- a/src/backend/utils/cache/catcache.c +++ b/src/backend/utils/cache/catcache.c @@ -141,6 +141,11 @@ GetCCHashEqFuncs(Oid keytype, PGFunction *hashfunc, RegProcedure *eqfunc) *eqfunc = F_TEXTEQ; break; + case TIDOID: /* Is this needed? */ + *hashfunc = hashtid; + + *eqfunc = F_TIDEQ; + break; case OIDOID: case REGPROCOID: case REGPROCEDUREOID: diff --git a/src/backend/utils/cache/lsyscache.c b/src/backend/utils/cache/lsyscache.c index 093da76..8b12328 100644 --- a/src/backend/utils/cache/lsyscache.c +++ b/src/backend/utils/cache/lsyscache.c @@ -24,6 +24,7 @@ #include "catalog/pg_amproc.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" +#include "catalog/pg_cstore.h" #include "catalog/pg_language.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" @@ -1685,6 +1686,35 @@ get_rel_name(Oid relid) return NULL; } + +/* + * get_cstore_name + * Returns the name of a given column store. + * + * Returns a palloc'd copy of the string, or NULL if no such column store. + * + * NOTE: since column store name is not unique, be wary of code that uses this + * for anything except preparing error messages. + */ +char * +get_cstore_name(Oid relid) +{ + HeapTuple tp; + + tp = SearchSysCache1(CSTOREOID, ObjectIdGetDatum(relid)); + if (HeapTupleIsValid(tp)) + { + Form_pg_cstore reltup = (Form_pg_cstore) GETSTRUCT(tp); + char *result; + + result = pstrdup(NameStr(reltup->cstname)); + ReleaseSysCache(tp); + return result; + } + else + return NULL; +} + /* * get_rel_namespace * diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 6b0c0b7..9030839 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -273,6 +273,7 @@ static void IndexSupportInitialize(oidvector *indclass, AttrNumber maxAttributeNumber); static OpClassCacheEnt *LookupOpclassInfo(Oid operatorClassOid, StrategyNumber numSupport); +static void RelationInitColumnStoreInfo(Relation rel); static void RelationCacheInitFileRemoveInDir(const char *tblspcpath); static void unlink_initfile(const char *initfilename); @@ -432,6 +433,7 @@ RelationParseRelOptions(Relation relation, HeapTuple tuple) case RELKIND_INDEX: case RELKIND_VIEW: case RELKIND_MATVIEW: + case RELKIND_COLUMN_STORE: break; default: return; @@ -479,6 +481,8 @@ RelationBuildTupleDesc(Relation relation) TupleConstr *constr; AttrDefault *attrdef = NULL; int ndef = 0; + int nphyatts = 0; + AttributeOrdering attorder; /* copy some fields from pg_class row to rd_att */ relation->rd_att->tdtypeid = relation->rd_rel->reltype; @@ -520,6 +524,17 @@ RelationBuildTupleDesc(Relation relation) */ need = relation->rd_rel->relnatts; + /* + * We maintain a enum to indicate if the logical column order matches the + * physical on-disk column order. + * + * Initially we'll assume the physical and logical orders match. In the + * loop below we'll adjust the value as required depending on what we + * discover about the column ordering. + */ + attorder = ATTRORDER_PHYSMATCHLOGICAL; + + MemSet(relation->rd_att->attrmap, 0, need * sizeof(AttrNumber)); while (HeapTupleIsValid(pg_attribute_tuple = systable_getnext(pg_attribute_scan))) { Form_pg_attribute attp; @@ -539,6 +554,31 @@ RelationBuildTupleDesc(Relation relation) if (attp->attnotnull) constr->has_not_null = true; + /* + * If we find any attributes where the the logical order does not match + * the phyiscal order, then we'll mark the TupleDesc with the + * appropriate AttributeOrdering. + */ + if (attp->attnum != attp->attphynum) + { + if (attp->attphynum == InvalidAttrNumber) + attorder = ATTRORDER_OFFHEAPATTRS; + + else if (attorder == ATTRORDER_PHYSMATCHLOGICAL) + attorder = ATTRORDER_OUTOFORDER; + } + + if (attp->attphynum != InvalidAttrNumber) + { + if (attp->attphynum < 0 || + attp->attphynum > relation->rd_rel->relnatts) + elog(ERROR, "invalid physical attribute number %d for %s", + attp->attphynum, RelationGetRelationName(relation)); + + relation->rd_att->attrmap[attp->attphynum - 1] = attp->attnum; + nphyatts++; + } + if (attp->atthasdef) { if (attrdef == NULL) @@ -555,6 +595,9 @@ RelationBuildTupleDesc(Relation relation) break; } + relation->rd_att->nphyatts = nphyatts; + relation->rd_att->tdattorder = attorder; + /* * end the scan and close the attribute relation */ @@ -1049,9 +1092,16 @@ RelationBuildDesc(Oid targetRelId, bool insertIt) /* * if it's an index, initialize index-related information */ - if (OidIsValid(relation->rd_rel->relam)) + if (relation->rd_rel->relkind == RELKIND_INDEX && + OidIsValid(relation->rd_rel->relam)) RelationInitIndexAccessInfo(relation); + /* + * if it's a column store, initialize pg_cstore data + */ + if (relation->rd_rel->relkind == RELKIND_COLUMN_STORE) + RelationInitColumnStoreInfo(relation); + /* extract reloptions if any */ RelationParseRelOptions(relation, pg_class_tuple); @@ -1525,6 +1575,27 @@ LookupOpclassInfo(Oid operatorClassOid, return opcentry; } +/* + * For a column store relation, initialize rd_cstore + */ +static void +RelationInitColumnStoreInfo(Relation rel) +{ + HeapTuple tuple; + MemoryContext oldcontext; + + tuple = SearchSysCache1(CSTOREOID, + ObjectIdGetDatum(RelationGetRelid(rel))); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for column store %u", + RelationGetRelid(rel)); + oldcontext = MemoryContextSwitchTo(CacheMemoryContext); + rel->rd_cstoretuple = heap_copytuple(tuple); + rel->rd_cstore = (Form_pg_cstore) GETSTRUCT(rel->rd_cstoretuple); + MemoryContextSwitchTo(oldcontext); + + ReleaseSysCache(tuple); +} /* * formrdesc @@ -1688,7 +1759,7 @@ formrdesc(const char *relationName, Oid relationReltype, */ if (IsBootstrapProcessingMode()) { - /* In bootstrap mode, we have no indexes */ + /* In bootstrap mode, we have no indexes or column stores */ relation->rd_rel->relhasindex = false; } else @@ -1697,6 +1768,9 @@ formrdesc(const char *relationName, Oid relationReltype, relation->rd_rel->relhasindex = true; } + /* this is used for catalogs only, so no column stores */ + relation->rd_rel->relhascstore = false; + /* * add new reldesc to relcache */ @@ -2009,6 +2083,8 @@ RelationDestroyRelation(Relation relation, bool remember_tupdesc) pfree(relation->rd_options); if (relation->rd_indextuple) pfree(relation->rd_indextuple); + if (relation->rd_cstoretuple) + pfree(relation->rd_cstoretuple); if (relation->rd_am) pfree(relation->rd_am); if (relation->rd_indexcxt) @@ -2019,6 +2095,8 @@ RelationDestroyRelation(Relation relation, bool remember_tupdesc) MemoryContextDelete(relation->rd_rsdesc->rscxt); if (relation->rd_fdwroutine) pfree(relation->rd_fdwroutine); + if (relation->rd_colstoreroutine) + pfree(relation->rd_colstoreroutine); pfree(relation); } @@ -3923,6 +4001,100 @@ RelationGetIndexList(Relation relation) } /* + * RelationGetColStoreList -- get a list of OIDs of colstores on this relation + * + * XXX Copy'n'paste of RelationGetIndexList(), simplified for colstores. + * + * The colstore list is created only if someone requests it. We scan pg_cstore + * to find relevant colstores, and add the list to the relcache entry so that + * we won't have to compute it again. Note that shared cache inval of a + * relcache entry will delete the old list and set rd_cstvalid to 0, + * so that we must recompute the colstore list on next request. This handles + * creation or deletion of a colstore. + * + * XXX At the moment we don't need the invalidation, because all the colstores + * are defined at table creation time, but if we ever decide to implement + * ALTER TABLE ... [ADD|DROP] COLUMN STORE, this will be handy. Also, this + * is just copy'n'paste programming using RelationGetIndexList(). + * + * XXX Currently there are no 'islive' or 'isvalid' flags for colstores. + * + * The returned list is guaranteed to be sorted in order by OID. This is + * needed by the executor, since for colstore types that we obtain exclusive + * locks on when updating the colstore, all backends must lock the colstores + * in the same order or we will get deadlocks (see ExecOpenColstores()). Any + * consistent ordering would do, but ordering by OID is easy. + * + * Since shared cache inval causes the relcache's copy of the list to go away, + * we return a copy of the list palloc'd in the caller's context. The caller + * may list_free() the returned list after scanning it. This is necessary + * since the caller will typically be doing syscache lookups on the relevant + * colstores, and syscache lookup could cause SI messages to be processed! + */ +List * +RelationGetColStoreList(Relation relation) +{ + Relation cstrel; + SysScanDesc cstscan; + ScanKeyData skey; + HeapTuple htup; + List *result; + List *oldlist; + MemoryContext oldcxt; + + /* Quick exit if we already computed the list. */ + if (relation->rd_cstvalid) + return list_copy(relation->rd_cstlist); + + /* + * We build the list we intend to return (in the caller's context) while + * doing the scan. After successfully completing the scan, we copy that + * list into the relcache entry. This avoids cache-context memory leakage + * if we get some sort of error partway through. + */ + result = NIL; + + /* Prepare to scan pg_cstore for entries having cstrelid = this rel. */ + ScanKeyInit(&skey, + Anum_pg_cstore_cstrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationGetRelid(relation))); + + cstrel = heap_open(CStoreRelationId, AccessShareLock); + cstscan = systable_beginscan(cstrel, CStoreCstRelidCstnameIndexId, true, + NULL, 1, &skey); + + while (HeapTupleIsValid(htup = systable_getnext(cstscan))) + { + Form_pg_cstore cstore = (Form_pg_cstore) GETSTRUCT(htup); + + /* + * TODO Ignore column stores that are being dropped (ALTER TABLE + * drop COLUMN STORE) here. + */ + + /* Add index's OID to result list in the proper order */ + result = insert_ordered_oid(result, cstore->cststoreid); + } + + systable_endscan(cstscan); + + heap_close(cstrel, AccessShareLock); + + /* Now save a copy of the completed list in the relcache entry. */ + oldcxt = MemoryContextSwitchTo(CacheMemoryContext); + oldlist = relation->rd_cstlist; + relation->rd_cstlist = list_copy(result); + relation->rd_cstvalid = true; + MemoryContextSwitchTo(oldcxt); + + /* Don't leak the old list, if there is one */ + list_free(oldlist); + + return result; +} + +/* * insert_ordered_oid * Insert a new Oid into a sorted list of Oids, preserving ordering * @@ -4875,6 +5047,7 @@ load_relcache_init_file(bool shared) rel->rd_exclprocs = NULL; rel->rd_exclstrats = NULL; rel->rd_fdwroutine = NULL; + rel->rd_colstoreroutine = NULL; /* * Reset transient-state fields in the relcache entry @@ -4894,6 +5067,8 @@ load_relcache_init_file(bool shared) rel->rd_createSubid = InvalidSubTransactionId; rel->rd_newRelfilenodeSubid = InvalidSubTransactionId; rel->rd_amcache = NULL; + rel->rd_cstvalid = 0; + rel->rd_cstlist = NIL; MemSet(&rel->pgstat_info, 0, sizeof(rel->pgstat_info)); /* diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index efce7b9..1c39625 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -32,6 +32,8 @@ #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_conversion.h" +#include "catalog/pg_cstore.h" +#include "catalog/pg_cstore_am.h" #include "catalog/pg_database.h" #include "catalog/pg_db_role_setting.h" #include "catalog/pg_default_acl.h" @@ -347,6 +349,39 @@ static const struct cachedesc cacheinfo[] = { }, 8 }, + {CStoreAmRelationId, /* CSTOREAMNAME */ + CStoreAmNameIndexId, + 1, + { + Anum_pg_cstore_am_cstamname, + 0, + 0, + 0 + }, + 8 + }, + {CStoreAmRelationId, /* CSTOREAMOID */ + CStoreAmOidIndexId, + 1, + { + ObjectIdAttributeNumber, + 0, + 0, + 0 + }, + 8 + }, + {CStoreRelationId, /* CSTOREOID */ + CStoreOidIndexId, + 1, + { + ObjectIdAttributeNumber, + 0, + 0, + 0 + }, + 16 + }, {DatabaseRelationId, /* DATABASEOID */ DatabaseOidIndexId, 1, diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index bb59bc2..0de8264 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1525,6 +1525,10 @@ describeOneTableDetails(const char *schemaname, printfPQExpBuffer(&title, _("Composite type \"%s.%s\""), schemaname, relationname); break; + case 'C': + printfPQExpBuffer(&title, _("Column store \"%s.%s\""), + schemaname, relationname); + break; case 'f': printfPQExpBuffer(&title, _("Foreign table \"%s.%s\""), schemaname, relationname); @@ -2534,6 +2538,30 @@ describeOneTableDetails(const char *schemaname, /* Tablespace info */ add_tablespace_footer(&cont, tableinfo.relkind, tableinfo.tablespace, true); + + if (pset.sversion >= 90600) + { + printfPQExpBuffer(&buf, "SELECT cstamname, cstname, (SELECT string_agg(attname,', ') FROM pg_attribute WHERE attrelid = cstrelid AND attnum = ANY(cstatts)) FROM pg_cstore AS cst, pg_class AS cl, pg_cstore_am AS am WHERE cl.oid = cst.cststoreid AND cl.relam = am.oid AND cstrelid = '%s'::regclass ORDER BY cstatts[0];", oid); + result = PSQLexec(buf.data); + if (!result) + goto error_return; + else + tuples = PQntuples(result); + + if (tuples > 0) + { + printTableAddFooter(&cont, _("Column stores:")); + for (i = 0; i < tuples; i++) + { + printfPQExpBuffer(&buf, " \"%s\" USING %s (%s)", + PQgetvalue(result, i, 1), + PQgetvalue(result, i, 0), + PQgetvalue(result, i, 2)); + printTableAddFooter(&cont, buf.data); + } + } + PQclear(result); + } } /* reloptions, if verbose */ diff --git a/src/include/access/hash.h b/src/include/access/hash.h index 97cb859..d5a4e42 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -271,6 +271,7 @@ extern Datum hashint2(PG_FUNCTION_ARGS); extern Datum hashint4(PG_FUNCTION_ARGS); extern Datum hashint8(PG_FUNCTION_ARGS); extern Datum hashoid(PG_FUNCTION_ARGS); +extern Datum hashtid(PG_FUNCTION_ARGS); extern Datum hashenum(PG_FUNCTION_ARGS); extern Datum hashfloat4(PG_FUNCTION_ARGS); extern Datum hashfloat8(PG_FUNCTION_ARGS); diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index 8dd530bd..d7ef372 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -708,7 +708,8 @@ struct MinimalTupleData ) \ : \ ( \ - att_isnull((attnum)-1, (tup)->t_data->t_bits) ? \ + att_isnull((tupleDesc)->attrs[(attnum) - 1]->attphynum-1, \ + (tup)->t_data->t_bits) ? \ ( \ (*(isnull) = true), \ (Datum)NULL \ @@ -719,8 +720,39 @@ struct MinimalTupleData ) \ ) \ ) + +#define fastgetlogattr(tup, attnum, tupleDesc, isnull) \ +( \ + AssertMacro((attnum) > 0), \ + (*(isnull) = false), \ + HeapTupleNoNulls(tup) ? \ + ( \ + (tupleDesc)->attrs[(attnum)-1]->attcacheoff >= 0 ? \ + ( \ + fetchatt((tupleDesc)->attrs[(attnum)-1], \ + (char *) (tup)->t_data + (tup)->t_data->t_hoff + \ + (tupleDesc)->attrs[(attnum)-1]->attcacheoff) \ + ) \ + : \ + nocachegetlogattr((tup), (attnum), (tupleDesc)) \ + ) \ + : \ + ( \ + att_isnull((attnum)-1, (tup)->t_data->t_bits) ? \ + ( \ + (*(isnull) = true), \ + (Datum)NULL \ + ) \ + : \ + ( \ + nocachegetlogattr((tup), (attnum), (tupleDesc)) \ + ) \ + ) \ +) + #else /* defined(DISABLE_COMPLEX_MACRO) */ + extern Datum fastgetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc, bool *isnull); #endif /* defined(DISABLE_COMPLEX_MACRO) */ @@ -736,16 +768,19 @@ extern Datum fastgetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc, * If the field in question has a NULL value, we return a zero Datum * and set *isnull == true. Otherwise, we set *isnull == false. * - * is the pointer to the heap tuple. is the attribute - * number of the column (field) caller wants. is a - * pointer to the structure describing the row and all its fields. + * is the pointer to the heap tuple. is the logical + * attribute number of the column (field) caller wants. This is translated + * into the phyical number internally using the tupleDesc. + * is a pointer to the structure describing the row and all + * its fields. * ---------------- */ #define heap_getattr(tup, attnum, tupleDesc, isnull) \ ( \ ((attnum) > 0) ? \ ( \ - ((attnum) > (int) HeapTupleHeaderGetNatts((tup)->t_data)) ? \ + ((tupleDesc)->attrs[(attnum) - 1]->attphynum > \ + (int) HeapTupleHeaderGetNatts((tup)->t_data)) ? \ ( \ (*(isnull) = true), \ (Datum)NULL \ @@ -757,10 +792,28 @@ extern Datum fastgetattr(HeapTuple tup, int attnum, TupleDesc tupleDesc, heap_getsysattr((tup), (attnum), (tupleDesc), (isnull)) \ ) +#define heap_getlogattr(tup, attnum, tupleDesc, isnull) \ + ( \ + ((attnum) > 0) ? \ + ( \ + ((attnum) > (int) HeapTupleHeaderGetNatts((tup)->t_data)) ? \ + ( \ + (*(isnull) = true), \ + (Datum)NULL \ + ) \ + : \ + fastgetlogattr((tup), (attnum), (tupleDesc), (isnull)) \ + ) \ + : \ + heap_getsysattr((tup), (attnum), (tupleDesc), (isnull)) \ + ) + /* prototypes for functions in common/heaptuple.c */ extern Size heap_compute_data_size(TupleDesc tupleDesc, Datum *values, bool *isnull); +extern Size heap_compute_logical_data_size(TupleDesc tupleDesc, + Datum *values, bool *isnull); extern void heap_fill_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull, char *data, Size data_size, @@ -768,6 +821,8 @@ extern void heap_fill_tuple(TupleDesc tupleDesc, extern bool heap_attisnull(HeapTuple tup, int attnum); extern Datum nocachegetattr(HeapTuple tup, int attnum, TupleDesc att); +extern Datum nocachegetlogattr(HeapTuple tup, int attnum, + TupleDesc att); extern Datum heap_getsysattr(HeapTuple tup, int attnum, TupleDesc tupleDesc, bool *isnull); extern HeapTuple heap_copytuple(HeapTuple tuple); @@ -775,6 +830,8 @@ extern void heap_copytuple_with_tuple(HeapTuple src, HeapTuple dest); extern Datum heap_copy_tuple_as_datum(HeapTuple tuple, TupleDesc tupleDesc); extern HeapTuple heap_form_tuple(TupleDesc tupleDescriptor, Datum *values, bool *isnull); +extern HeapTuple heap_form_logical_tuple(TupleDesc tupleDescriptor, + Datum *values, bool *isnull); extern HeapTuple heap_modify_tuple(HeapTuple tuple, TupleDesc tupleDesc, Datum *replValues, diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h index 91b0034..b0ec0e5 100644 --- a/src/include/access/tupdesc.h +++ b/src/include/access/tupdesc.h @@ -44,6 +44,32 @@ typedef struct tupleConstr } TupleConstr; /* + * This enum describes the attribute ordering within a Tuple. + * A relation may have the logical column ordering vary from the actual + * physical (on disk) order. In some cases this can allow a smaller on-disk + * footprint of a tuple by reducing the padding between attributes. + * It is also possible for attributes of a logical relation not to be stored + * in the heap at all. + * + * ATTRORDER_PHYSMATCHLOGICAL: + * The phyiscal attribute order is the same as the logical one, and there's no + * off-heap attributes. + * + * ATTRORDER_OUTOFORDER: + * Logical attribute order does not match the physical attribute order. There + * are no off-heap attributes. + * + * ATTRORDER_OFFHEAPATTRS: + * Contains off-heap attributes and heap attributes may be out of order. + */ +typedef enum AttributeOrdering +{ + ATTRORDER_PHYSMATCHLOGICAL, + ATTRORDER_OUTOFORDER, + ATTRORDER_OFFHEAPATTRS +} AttributeOrdering; + +/* * This struct is passed around within the backend to describe the structure * of tuples. For tuples coming from on-disk relations, the information is * collected from the pg_attribute, pg_attrdef, and pg_constraint catalogs. @@ -70,13 +96,16 @@ typedef struct tupleConstr */ typedef struct tupleDesc { - int natts; /* number of attributes in the tuple */ + int natts; /* number of logical attributes in the tuple */ + int nphyatts; /* number of physical attributes in the tuple */ + AttrNumber *attrmap; /* map of attnum indexed by attphynum-1 */ Form_pg_attribute *attrs; /* attrs[N] is a pointer to the description of Attribute Number N+1 */ TupleConstr *constr; /* constraints, or NULL if none */ Oid tdtypeid; /* composite type ID for tuple type */ int32 tdtypmod; /* typmod for tuple type */ bool tdhasoid; /* tuple has oid attribute in its header */ + AttributeOrdering tdattorder; /* Attibutes order type. See comment above */ int tdrefcount; /* reference count, or -1 if not counting */ } *TupleDesc; @@ -112,6 +141,8 @@ extern void DecrTupleDescRefCount(TupleDesc tupdesc); extern bool equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2); +extern void TupleDescCacheReset(TupleDesc desc); + extern void TupleDescInitEntry(TupleDesc desc, AttrNumber attributeNumber, const char *attributeName, @@ -123,7 +154,7 @@ extern void TupleDescInitEntryCollation(TupleDesc desc, AttrNumber attributeNumber, Oid collationid); -extern TupleDesc BuildDescForRelation(List *schema); +extern TupleDesc BuildDescForRelation(List *schema, List **colstores); extern TupleDesc BuildDescFromLists(List *names, List *types, List *typmods, List *collations); diff --git a/src/include/catalog/colstore.h b/src/include/catalog/colstore.h new file mode 100644 index 0000000..c49fe5e --- /dev/null +++ b/src/include/catalog/colstore.h @@ -0,0 +1,50 @@ +#ifndef COLSTORE_H +#define COLSTORE_H + +#include "nodes/execnodes.h" +#include "nodes/nodes.h" +#include "nodes/parsenodes.h" +#include "nodes/execnodes.h" +#include "nodes/pg_list.h" +#include "utils/relcache.h" + +/* + * When creating a table with column store declarations, this struct + * carries the necessary info. + * + * We're catering for several different cases here: (a) column store + * declarations as column constraints (attnum and cstoreClause are both set, + * attnums and cstoreOid are invalid), (b) declarations as table constraint + * (attnum is invalid but cstoreClause is set; attnums and cstoreOid are + * invalid), and (c) store definitions inherited from parent relations (attnum + * and cstoreClause are both invalid, attnums list the attribute numbers and + * cstoreOid is the OID of the pg_cstore entry of the column store for the + * parent relation. Note that the cstatts data from the parent's entry must be + * ignored in favor of the attnum list given here.) + */ +typedef struct ColumnStoreClauseInfo +{ + AttrNumber attnum; + ColumnStoreClause *cstoreClause; + List *attnums; + Oid cstoreOid; +} ColumnStoreClauseInfo; + + +extern List *DetermineColumnStores(TupleDesc tupdesc, List *decl_cstores, + List *inh_cstores, Oid tablespaceId); +extern void CreateColumnStores(Relation rel, List *colstores); +extern List *CloneColumnStores(Relation rel); +extern Oid get_relation_cstore_oid(Oid relid, const char *cstore_name, + bool missing_ok); +extern void RemoveColstoreById(Oid cstoreOid); +extern Oid GetColumnStoreAMByName(char *cstamname, bool missing_ok); + +/* XXX these are temporary hacks */ +extern void FormColumnStoreDatum(ColumnStoreHandler *handler, + HeapTuple tuple, TupleDesc tupdesc, + Datum *values, bool *isnull); +extern HeapTuple FilterHeapTuple(ResultRelInfo *resultRelInfo, HeapTuple tuple, + TupleDesc *heapdesc); + +#endif /* COLSTORE_H */ diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index fbcf904..71a28c3 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -138,6 +138,7 @@ typedef enum ObjectClass OCLASS_AMPROC, /* pg_amproc */ OCLASS_REWRITE, /* pg_rewrite */ OCLASS_TRIGGER, /* pg_trigger */ + OCLASS_COLSTORE, /* pg_cstore */ OCLASS_SCHEMA, /* pg_namespace */ OCLASS_TSPARSER, /* pg_ts_parser */ OCLASS_TSDICT, /* pg_ts_dict */ diff --git a/src/include/catalog/heap.h b/src/include/catalog/heap.h index e6ac394..27a065c 100644 --- a/src/include/catalog/heap.h +++ b/src/include/catalog/heap.h @@ -60,6 +60,7 @@ extern Oid heap_create_with_catalog(const char *relname, Oid ownerid, TupleDesc tupdesc, List *cooked_constraints, + List *colstores, char relkind, char relpersistence, bool shared_relation, @@ -95,6 +96,12 @@ extern void InsertPgClassTuple(Relation pg_class_desc, Datum relacl, Datum reloptions); +extern void AddNewAttributeTuples(Oid new_rel_oid, + TupleDesc tupdesc, + char relkind, + bool oidislocal, + int oidinhcount); + extern List *AddRelationNewConstraints(Relation rel, List *newColDefaults, List *newConstraints, diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index c38958d..ec6a5a2 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -130,6 +130,18 @@ DECLARE_UNIQUE_INDEX(pg_conversion_name_nsp_index, 2669, on pg_conversion using DECLARE_UNIQUE_INDEX(pg_conversion_oid_index, 2670, on pg_conversion using btree(oid oid_ops)); #define ConversionOidIndexId 2670 +DECLARE_UNIQUE_INDEX(pg_cstore_oid_index, 3398, on pg_cstore using btree(oid oid_ops)); +#define CStoreOidIndexId 3398 +DECLARE_INDEX(pg_cstore_cststoreid_index, 3399, on pg_cstore using btree(cststoreid oid_ops)); +#define CStoreStoreOidIndexId 3399 +DECLARE_UNIQUE_INDEX(pg_cstore_cstrelid_cstname_index, 3400, on pg_cstore using btree(cstrelid oid_ops, cstname name_ops)); +#define CStoreCstRelidCstnameIndexId 3400 + +DECLARE_UNIQUE_INDEX(pg_cstore_am_oid_index, 3394, on pg_cstore_am using btree(oid oid_ops)); +#define CStoreAmOidIndexId 3394 +DECLARE_UNIQUE_INDEX(pg_cstore_am_name_index, 3395, on pg_cstore_am using btree(cstamname name_ops)); +#define CStoreAmNameIndexId 3395 + DECLARE_UNIQUE_INDEX(pg_database_datname_index, 2671, on pg_database using btree(datname name_ops)); #define DatabaseNameIndexId 2671 DECLARE_UNIQUE_INDEX(pg_database_oid_index, 2672, on pg_database using btree(oid oid_ops)); diff --git a/src/include/catalog/pg_amop.h b/src/include/catalog/pg_amop.h index da5fe9d..03ad90b 100644 --- a/src/include/catalog/pg_amop.h +++ b/src/include/catalog/pg_amop.h @@ -557,6 +557,8 @@ DATA(insert ( 1985 829 829 1 s 1220 405 0 )); DATA(insert ( 1987 19 19 1 s 93 405 0 )); /* oid_ops */ DATA(insert ( 1990 26 26 1 s 607 405 0 )); +/* tid_ops */ +DATA(insert ( 3361 27 27 1 s 387 405 0 )); /* oidvector_ops */ DATA(insert ( 1992 30 30 1 s 649 405 0 )); /* text_ops */ diff --git a/src/include/catalog/pg_amproc.h b/src/include/catalog/pg_amproc.h index 7db2015..bc93507 100644 --- a/src/include/catalog/pg_amproc.h +++ b/src/include/catalog/pg_amproc.h @@ -160,6 +160,7 @@ DATA(insert ( 1983 1186 1186 1 1697 )); DATA(insert ( 1985 829 829 1 399 )); DATA(insert ( 1987 19 19 1 455 )); DATA(insert ( 1990 26 26 1 453 )); +DATA(insert ( 3361 27 27 1 3360 )); DATA(insert ( 1992 30 30 1 457 )); DATA(insert ( 1995 25 25 1 400 )); DATA(insert ( 1997 1083 1083 1 1688 )); diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h index f0b28b0..f2f520a 100644 --- a/src/include/catalog/pg_attribute.h +++ b/src/include/catalog/pg_attribute.h @@ -78,6 +78,12 @@ CATALOG(pg_attribute,1249) BKI_BOOTSTRAP BKI_WITHOUT_OIDS BKI_ROWTYPE_OID(75) BK int16 attnum; /* + * Physical location in the heap, or 0 if attribute is not stored in + * the heap. + */ + int16 attphynum; + + /* * attndims is the declared number of dimensions, if an array type, * otherwise zero. */ @@ -188,28 +194,29 @@ typedef FormData_pg_attribute *Form_pg_attribute; * ---------------- */ -#define Natts_pg_attribute 21 +#define Natts_pg_attribute 22 #define Anum_pg_attribute_attrelid 1 #define Anum_pg_attribute_attname 2 #define Anum_pg_attribute_atttypid 3 #define Anum_pg_attribute_attstattarget 4 #define Anum_pg_attribute_attlen 5 #define Anum_pg_attribute_attnum 6 -#define Anum_pg_attribute_attndims 7 -#define Anum_pg_attribute_attcacheoff 8 -#define Anum_pg_attribute_atttypmod 9 -#define Anum_pg_attribute_attbyval 10 -#define Anum_pg_attribute_attstorage 11 -#define Anum_pg_attribute_attalign 12 -#define Anum_pg_attribute_attnotnull 13 -#define Anum_pg_attribute_atthasdef 14 -#define Anum_pg_attribute_attisdropped 15 -#define Anum_pg_attribute_attislocal 16 -#define Anum_pg_attribute_attinhcount 17 -#define Anum_pg_attribute_attcollation 18 -#define Anum_pg_attribute_attacl 19 -#define Anum_pg_attribute_attoptions 20 -#define Anum_pg_attribute_attfdwoptions 21 +#define Anum_pg_attribute_attphynum 7 +#define Anum_pg_attribute_attndims 8 +#define Anum_pg_attribute_attcacheoff 9 +#define Anum_pg_attribute_atttypmod 10 +#define Anum_pg_attribute_attbyval 11 +#define Anum_pg_attribute_attstorage 12 +#define Anum_pg_attribute_attalign 13 +#define Anum_pg_attribute_attnotnull 14 +#define Anum_pg_attribute_atthasdef 15 +#define Anum_pg_attribute_attisdropped 16 +#define Anum_pg_attribute_attislocal 17 +#define Anum_pg_attribute_attinhcount 18 +#define Anum_pg_attribute_attcollation 19 +#define Anum_pg_attribute_attacl 20 +#define Anum_pg_attribute_attoptions 21 +#define Anum_pg_attribute_attfdwoptions 22 /* ---------------- diff --git a/src/include/catalog/pg_class.h b/src/include/catalog/pg_class.h index 06d287e..aff4489 100644 --- a/src/include/catalog/pg_class.h +++ b/src/include/catalog/pg_class.h @@ -49,6 +49,7 @@ CATALOG(pg_class,1259) BKI_BOOTSTRAP BKI_ROWTYPE_OID(83) BKI_SCHEMA_MACRO * up-to-date) */ Oid reltoastrelid; /* OID of toast table; 0 if none */ bool relhasindex; /* T if has (or has had) any indexes */ + bool relhascstore; /* T if has (or has had) any column stores */ bool relisshared; /* T if shared across databases */ char relpersistence; /* see RELPERSISTENCE_xxx constants below */ char relkind; /* see RELKIND_xxx constants below */ @@ -96,7 +97,7 @@ typedef FormData_pg_class *Form_pg_class; * ---------------- */ -#define Natts_pg_class 31 +#define Natts_pg_class 32 #define Anum_pg_class_relname 1 #define Anum_pg_class_relnamespace 2 #define Anum_pg_class_reltype 3 @@ -110,24 +111,25 @@ typedef FormData_pg_class *Form_pg_class; #define Anum_pg_class_relallvisible 11 #define Anum_pg_class_reltoastrelid 12 #define Anum_pg_class_relhasindex 13 -#define Anum_pg_class_relisshared 14 -#define Anum_pg_class_relpersistence 15 -#define Anum_pg_class_relkind 16 -#define Anum_pg_class_relnatts 17 -#define Anum_pg_class_relchecks 18 -#define Anum_pg_class_relhasoids 19 -#define Anum_pg_class_relhaspkey 20 -#define Anum_pg_class_relhasrules 21 -#define Anum_pg_class_relhastriggers 22 -#define Anum_pg_class_relhassubclass 23 -#define Anum_pg_class_relrowsecurity 24 -#define Anum_pg_class_relforcerowsecurity 25 -#define Anum_pg_class_relispopulated 26 -#define Anum_pg_class_relreplident 27 -#define Anum_pg_class_relfrozenxid 28 -#define Anum_pg_class_relminmxid 29 -#define Anum_pg_class_relacl 30 -#define Anum_pg_class_reloptions 31 +#define Anum_pg_class_relhascstore 14 +#define Anum_pg_class_relisshared 15 +#define Anum_pg_class_relpersistence 16 +#define Anum_pg_class_relkind 17 +#define Anum_pg_class_relnatts 18 +#define Anum_pg_class_relchecks 19 +#define Anum_pg_class_relhasoids 20 +#define Anum_pg_class_relhaspkey 21 +#define Anum_pg_class_relhasrules 22 +#define Anum_pg_class_relhastriggers 23 +#define Anum_pg_class_relhassubclass 24 +#define Anum_pg_class_relrowsecurity 25 +#define Anum_pg_class_relforcerowsecurity 26 +#define Anum_pg_class_relispopulated 27 +#define Anum_pg_class_relreplident 28 +#define Anum_pg_class_relfrozenxid 29 +#define Anum_pg_class_relminmxid 30 +#define Anum_pg_class_relacl 31 +#define Anum_pg_class_reloptions 32 /* ---------------- * initial contents of pg_class @@ -142,13 +144,13 @@ typedef FormData_pg_class *Form_pg_class; * Note: "3" in the relfrozenxid column stands for FirstNormalTransactionId; * similarly, "1" in relminmxid stands for FirstMultiXactId */ -DATA(insert OID = 1247 ( pg_type PGNSP 71 0 PGUID 0 0 0 0 0 0 0 f f p r 30 0 t f f f f f f t n 3 1 _null_ _null_ )); +DATA(insert OID = 1247 ( pg_type PGNSP 71 0 PGUID 0 0 0 0 0 0 0 f f f p r 30 0 t f f f f f f t n 3 1 _null_ _null_ )); DESCR(""); -DATA(insert OID = 1249 ( pg_attribute PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f p r 21 0 f f f f f f f t n 3 1 _null_ _null_ )); +DATA(insert OID = 1249 ( pg_attribute PGNSP 75 0 PGUID 0 0 0 0 0 0 0 f f f p r 22 0 f f f f f f f t n 3 1 _null_ _null_ )); DESCR(""); -DATA(insert OID = 1255 ( pg_proc PGNSP 81 0 PGUID 0 0 0 0 0 0 0 f f p r 29 0 t f f f f f f t n 3 1 _null_ _null_ )); +DATA(insert OID = 1255 ( pg_proc PGNSP 81 0 PGUID 0 0 0 0 0 0 0 f f f p r 29 0 t f f f f f f t n 3 1 _null_ _null_ )); DESCR(""); -DATA(insert OID = 1259 ( pg_class PGNSP 83 0 PGUID 0 0 0 0 0 0 0 f f p r 31 0 t f f f f f f t n 3 1 _null_ _null_ )); +DATA(insert OID = 1259 ( pg_class PGNSP 83 0 PGUID 0 0 0 0 0 0 0 f f f p r 32 0 t f f f f f f t n 3 1 _null_ _null_ )); DESCR(""); @@ -160,6 +162,7 @@ DESCR(""); #define RELKIND_COMPOSITE_TYPE 'c' /* composite type */ #define RELKIND_FOREIGN_TABLE 'f' /* foreign table */ #define RELKIND_MATVIEW 'm' /* materialized view */ +#define RELKIND_COLUMN_STORE 'C' /* columnar store */ #define RELPERSISTENCE_PERMANENT 'p' /* regular table */ #define RELPERSISTENCE_UNLOGGED 'u' /* unlogged permanent table */ diff --git a/src/include/catalog/pg_cstore.h b/src/include/catalog/pg_cstore.h new file mode 100644 index 0000000..4ef8d70 --- /dev/null +++ b/src/include/catalog/pg_cstore.h @@ -0,0 +1,60 @@ +/*------------------------------------------------------------------------- + * + * pg_cstore.h + * definition of column stores - groups of attributes stored in + * columnar orientation, along with the relation's initial contents. + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_cstore.h + * + * NOTES + * the genbki.pl script reads this file and generates .bki + * information from the DATA() statements. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_CSTORE_H +#define PG_CSTORE_H + +#include "catalog/genbki.h" + +/* ---------------- + * pg_cstore definition. cpp turns this into + * typedef struct FormData_pg_cstore + * ---------------- + */ +#define CStoreRelationId 3397 + +CATALOG(pg_cstore,3397) +{ + NameData cstname; /* name of the colstore */ + Oid cstrelid; /* relation containing this cstore */ + Oid cststoreid; /* pg_class OID of the cstore itself */ + int16 cstnatts; /* number of attributes in the cstore */ + + /* variable-length fields start here, but we allow direct access to cstatts */ + int2vector cstatts; /* column numbers of cols in this store */ + +} FormData_pg_cstore; + +/* ---------------- + * Form_pg_cstore corresponds to a pointer to a tuple with + * the format of pg_cstore relation. + * ---------------- + */ +typedef FormData_pg_cstore *Form_pg_cstore; + +/* ---------------- + * compiler constants for pg_cstore + * ---------------- + */ +#define Natts_pg_cstore 5 +#define Anum_pg_cstore_cstname 1 +#define Anum_pg_cstore_cstrelid 2 +#define Anum_pg_cstore_cststoreid 3 +#define Anum_pg_cstore_cstnatts 4 +#define Anum_pg_cstore_cstatts 5 + +#endif /* PG_CSTORE_H */ diff --git a/src/include/catalog/pg_cstore_am.h b/src/include/catalog/pg_cstore_am.h new file mode 100644 index 0000000..232b0b2 --- /dev/null +++ b/src/include/catalog/pg_cstore_am.h @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------- + * + * pg_cstore_am.h + * definition of the system "cstore access method" relation + * (pg_cstore_am) along with the relation's initial contents. + * + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_cstore_am.h + * + * NOTES + * the genbki.pl script reads this file and generates .bki + * information from the DATA() statements. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_CSTORE_AM_H +#define PG_CSTORE_AM_H + +#include "catalog/genbki.h" + +/* ---------------- + * pg_cstore_am definition. cpp turns this into + * typedef struct FormData_pg_cstore_am + * ---------------- + */ +#define CStoreAmRelationId 3396 + +CATALOG(pg_cstore_am,3396) +{ + NameData cstamname; /* column store am name */ + Oid cstamhandler; /* handler function */ + +} FormData_pg_cstore_am; + +/* ---------------- + * Form_pg_cstore_am corresponds to a pointer to a tuple with + * the format of pg_cstore_am relation. + * ---------------- + */ +typedef FormData_pg_cstore_am *Form_pg_cstore_am; + +/* ---------------- + * compiler constants for pg_cstore_am + * ---------------- + */ +#define Natts_pg_cstore_am 2 +#define Anum_pg_cstore_am_cstamname 1 +#define Anum_pg_cstore_am_cstamhandler 2 + +#endif /* PG_CSTORE_AM_H */ diff --git a/src/include/catalog/pg_namespace.h b/src/include/catalog/pg_namespace.h index 2c8887c..d1c55d6 100644 --- a/src/include/catalog/pg_namespace.h +++ b/src/include/catalog/pg_namespace.h @@ -75,6 +75,9 @@ DESCR("reserved schema for TOAST tables"); DATA(insert OID = 2200 ( "public" PGUID _null_ )); DESCR("standard public schema"); #define PG_PUBLIC_NAMESPACE 2200 +DATA(insert OID = 9 ( "pg_colstore" PGUID _null_ )); +DESCR("reserved schema for column stores"); +#define PG_COLSTORE_NAMESPACE 9 /* diff --git a/src/include/catalog/pg_opclass.h b/src/include/catalog/pg_opclass.h index e7b3148..958d02e 100644 --- a/src/include/catalog/pg_opclass.h +++ b/src/include/catalog/pg_opclass.h @@ -169,6 +169,7 @@ DATA(insert ( 405 bool_ops PGNSP PGUID 2222 16 t 0 )); DATA(insert ( 405 bytea_ops PGNSP PGUID 2223 17 t 0 )); DATA(insert ( 405 int2vector_ops PGNSP PGUID 2224 22 t 0 )); DATA(insert ( 403 tid_ops PGNSP PGUID 2789 27 t 0 )); +DATA(insert ( 405 tid_ops PGNSP PGUID 3361 27 t 0 )); DATA(insert ( 405 xid_ops PGNSP PGUID 2225 28 t 0 )); DATA(insert ( 405 cid_ops PGNSP PGUID 2226 29 t 0 )); DATA(insert ( 405 abstime_ops PGNSP PGUID 2227 702 t 0 )); diff --git a/src/include/catalog/pg_operator.h b/src/include/catalog/pg_operator.h index e79ce57..26bc5b0 100644 --- a/src/include/catalog/pg_operator.h +++ b/src/include/catalog/pg_operator.h @@ -161,7 +161,7 @@ DESCR("equal"); DATA(insert OID = 386 ( "=" PGNSP PGUID b f t 22 22 16 386 0 int2vectoreq eqsel eqjoinsel )); DESCR("equal"); -DATA(insert OID = 387 ( "=" PGNSP PGUID b t f 27 27 16 387 402 tideq eqsel eqjoinsel )); +DATA(insert OID = 387 ( "=" PGNSP PGUID b t t 27 27 16 387 402 tideq eqsel eqjoinsel )); DESCR("equal"); #define TIDEqualOperator 387 DATA(insert OID = 402 ( "<>" PGNSP PGUID b f f 27 27 16 402 387 tidne neqsel neqjoinsel )); diff --git a/src/include/catalog/pg_opfamily.h b/src/include/catalog/pg_opfamily.h index acbc100..8a117fd 100644 --- a/src/include/catalog/pg_opfamily.h +++ b/src/include/catalog/pg_opfamily.h @@ -118,6 +118,7 @@ DATA(insert OID = 2222 ( 405 bool_ops PGNSP PGUID )); DATA(insert OID = 2223 ( 405 bytea_ops PGNSP PGUID )); DATA(insert OID = 2224 ( 405 int2vector_ops PGNSP PGUID )); DATA(insert OID = 2789 ( 403 tid_ops PGNSP PGUID )); +DATA(insert OID = 3361 ( 405 tid_ops PGNSP PGUID )); DATA(insert OID = 2225 ( 405 xid_ops PGNSP PGUID )); DATA(insert OID = 2226 ( 405 cid_ops PGNSP PGUID )); DATA(insert OID = 2227 ( 405 abstime_ops PGNSP PGUID )); diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index d8640db..06ce4e2 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -736,6 +736,8 @@ DATA(insert OID = 452 ( hashfloat8 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 DESCR("hash"); DATA(insert OID = 453 ( hashoid PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "26" _null_ _null_ _null_ _null_ _null_ hashoid _null_ _null_ _null_ )); DESCR("hash"); +DATA(insert OID = 3360 ( hashtid PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "27" _null_ _null_ _null_ _null_ _null_ hashtid _null_ _null_ _null_ )); +DESCR("hash"); DATA(insert OID = 454 ( hashchar PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "18" _null_ _null_ _null_ _null_ _null_ hashchar _null_ _null_ _null_ )); DESCR("hash"); DATA(insert OID = 455 ( hashname PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "19" _null_ _null_ _null_ _null_ _null_ hashname _null_ _null_ _null_ )); @@ -3668,6 +3670,8 @@ DATA(insert OID = 2997 ( pg_table_size PGNSP PGUID 12 1 0 0 0 f f f f t f v s DESCR("disk space usage for the specified table, including TOAST, free space and visibility map"); DATA(insert OID = 2998 ( pg_indexes_size PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 20 "2205" _null_ _null_ _null_ _null_ _null_ pg_indexes_size _null_ _null_ _null_ )); DESCR("disk space usage for all indexes attached to the specified table"); +DATA(insert OID = 3350 ( pg_column_stores_size PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 20 "2205" _null_ _null_ _null_ _null_ _null_ pg_column_stores_size _null_ _null_ _null_ )); +DESCR("disk space usage for all column stores attached to the specified table"); DATA(insert OID = 2999 ( pg_relation_filenode PGNSP PGUID 12 1 0 0 0 f f f f t f s s 1 0 26 "2205" _null_ _null_ _null_ _null_ _null_ pg_relation_filenode _null_ _null_ _null_ )); DESCR("filenode identifier of relation"); DATA(insert OID = 3454 ( pg_filenode_relation PGNSP PGUID 12 1 0 0 0 f f f f t f s s 2 0 2205 "26 26" _null_ _null_ _null_ _null_ _null_ pg_filenode_relation _null_ _null_ _null_ )); @@ -3742,6 +3746,10 @@ DATA(insert OID = 3311 ( tsm_handler_in PGNSP PGUID 12 1 0 0 0 f f f f f f i s DESCR("I/O"); DATA(insert OID = 3312 ( tsm_handler_out PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 2275 "3310" _null_ _null_ _null_ _null_ _null_ tsm_handler_out _null_ _null_ _null_ )); DESCR("I/O"); +DATA(insert OID = 3352 ( cstore_handler_in PGNSP PGUID 12 1 0 0 0 f f f f f f i s 1 0 3351 "2275" _null_ _null_ _null_ _null_ _null_ cstore_handler_in _null_ _null_ _null_ )); +DESCR("I/O"); +DATA(insert OID = 3353 ( cstore_handler_out PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 2275 "3351" _null_ _null_ _null_ _null_ _null_ cstore_handler_out _null_ _null_ _null_ )); +DESCR("I/O"); /* tablesample method handlers */ DATA(insert OID = 3313 ( bernoulli PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 3310 "2281" _null_ _null_ _null_ _null_ _null_ tsm_bernoulli_handler _null_ _null_ _null_ )); @@ -3749,6 +3757,11 @@ DESCR("BERNOULLI tablesample method handler"); DATA(insert OID = 3314 ( system PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 3310 "2281" _null_ _null_ _null_ _null_ _null_ tsm_system_handler _null_ _null_ _null_ )); DESCR("SYSTEM tablesample method handler"); +/* column stores */ +/* XXX parallel safe? */ +DATA(insert OID = 4337 ( vertical_cstore_handler PGNSP PGUID 12 1 0 0 0 f f f f t f i s 0 0 3351 "" _null_ _null_ _null_ _null_ _null_ vertical_cstore_handler _null_ _null_ _null_ )); +DESCR("vertical-partitioning column store handler"); + /* cryptographic */ DATA(insert OID = 2311 ( md5 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 25 "25" _null_ _null_ _null_ _null_ _null_ md5_text _null_ _null_ _null_ )); DESCR("MD5 hash"); diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 7dc95c8..716d487 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -696,6 +696,8 @@ DATA(insert OID = 3115 ( fdw_handler PGNSP PGUID 4 t p P f t \054 0 0 0 fdw_han #define FDW_HANDLEROID 3115 DATA(insert OID = 3310 ( tsm_handler PGNSP PGUID 4 t p P f t \054 0 0 0 tsm_handler_in tsm_handler_out - - - - - i p f 0 -1 0 0 _null_ _null_ _null_ )); #define TSM_HANDLEROID 3310 +DATA(insert OID = 3351 ( cstore_handler PGNSP PGUID 4 t p P f t \054 0 0 0 cstore_handler_in cstore_handler_out - - - - - i p f 0 -1 0 0 _null_ _null_ _null_ )); +#define CSTORE_HANDLEROID 3351 DATA(insert OID = 3831 ( anyrange PGNSP PGUID -1 f p P f t \054 0 0 0 anyrange_in anyrange_out - - - - - d x f 0 -1 0 0 _null_ _null_ _null_ )); #define ANYRANGEOID 3831 diff --git a/src/include/colstore/colstoreapi.h b/src/include/colstore/colstoreapi.h new file mode 100644 index 0000000..b94d93f --- /dev/null +++ b/src/include/colstore/colstoreapi.h @@ -0,0 +1,107 @@ +/*------------------------------------------------------------------------- + * + * colstoreapi.h + * API for column store implementations + * + * Copyright (c) 2010-2015, PostgreSQL Global Development Group + * + * src/include/colstore/colstoreapi.h + * + *------------------------------------------------------------------------- + */ +#ifndef COLSTOREAPI_H +#define COLSTOREAPI_H + +#include "nodes/execnodes.h" +#include "nodes/relation.h" + +typedef void (*ExecColumnStoreOpen_function) (ColumnStoreHandler *handler, + bool for_write, Snapshot snapshot); + +typedef void (*ExecColumnStoreInsert_function) (ColumnStoreHandler *handler, + Datum *values, bool *nulls, CommandId cid); + +typedef void (*ExecColumnStoreBatchInsert_function) (ColumnStoreHandler *handler, + int nrows, Datum **values, bool **nulls, + CommandId cid); + +typedef TupleTableSlot *(*ExecColumnStoreScanNext_function) (ColumnStoreHandler *handler, + TupleTableSlot *slot); + +typedef void (*ExecColumnStoreEndScan_function) (ColumnStoreHandler *handler); + +typedef void (*ExecColumnStoreRescan_function) (ColumnStoreHandler *handler); + +typedef void (*ExecColumnStoreMarkPos_function) (ColumnStoreHandler *handler); + +typedef void (*ExecColumnStoreRestrPos_function) (ColumnStoreHandler *handler); + +typedef void (*ExecColumnStoreClose_function) (ColumnStoreHandler *handler); + +typedef void (*ExecColumnStoreTruncate_function) (Relation rel); + +typedef int (*ExecColumnStoreSample_function) (Relation onerel, int elevel, + HeapTuple *rows, int targrows, + double *totalrows, + double *totaldeadrows); +/* + * ColumnStoreRoutine is the struct returned by a column store's handler + * function. It provides pointers to the callback functions needed by the + * planner and executor. + * + * More function pointers are likely to be added in the future. Therefore + * it's recommended that the handler initialize the struct with + * makeNode(ColumnStoreRoutine) so that all fields are set to NULL. This will + * ensure that no fields are accidentally left undefined. + */ +typedef struct ColumnStoreRoutine +{ + NodeTag type; + + /* open a column store, return opaque pointer */ + ExecColumnStoreOpen_function ExecColumnStoreOpen; + + /* insert a single row into the column store */ + ExecColumnStoreInsert_function ExecColumnStoreInsert; + + /* insert a batch of rows into the column store */ + ExecColumnStoreBatchInsert_function ExecColumnStoreBatchInsert; + + /* return next tuple in scan or NULL when there's no more */ + ExecColumnStoreScanNext_function ExecColumnStoreScanNext; + + /* end column store scan and free any resources used for scan */ + ExecColumnStoreEndScan_function ExecColumnStoreEndScan; + + /* reset scan position back to the first tuple */ + ExecColumnStoreRescan_function ExecColumnStoreRescan; + + /* mark the current position of a column store scan */ + ExecColumnStoreMarkPos_function ExecColumnStoreMarkPos; + + /* restore scan position to the previously marked position */ + ExecColumnStoreRestrPos_function ExecColumnStoreRestrPos; + + /* close a column store */ + ExecColumnStoreClose_function ExecColumnStoreClose; + + /* truncate column store */ + ExecColumnStoreTruncate_function ExecColumnStoreTruncate; + + /* Populated an array of sample rows */ + ExecColumnStoreSample_function ExecColumnStoreSample; +} ColumnStoreRoutine; + + +/* prototypes for functions in catalog/colstore.c */ +extern ColumnStoreHandler *BuildColumnStoreHandler(Relation cstore, + bool for_write, LOCKMODE lockmode, Snapshot snapshot); +extern void CloseColumnStore(ColumnStoreHandler *handler); + +extern Oid GetColumnStoreHandlerByRelId(Oid relid); +extern ColumnStoreRoutine *GetColumnStoreRoutine(Oid csthandler); +extern ColumnStoreRoutine *GetColumnStoreRoutineByRelId(Oid relid); +extern ColumnStoreRoutine *GetColumnStoreRoutineForRelation(Relation relation, + bool makecopy); + +#endif /* COLSTOREAPI_H */ diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index adae296..cd4bed3 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -132,6 +132,7 @@ extern Oid RemoveUserMapping(DropUserMappingStmt *stmt); extern void RemoveUserMappingById(Oid umId); extern void CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid); extern void ImportForeignSchema(ImportForeignSchemaStmt *stmt); +extern ObjectAddress CreateColumnStoreAM(CreateColumnStoreAMStmt *stmt); extern Datum transformGenericOptions(Oid catalogId, Datum oldOptions, List *options, diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index e3a31af..13095ff 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -203,4 +203,9 @@ extern double anl_random_fract(void); extern double anl_init_selection_state(int n); extern double anl_get_next_S(double t, int n, double *stateptr); +extern int acquire_sample_rows(Relation onerel, int elevel, HeapTuple *rows, + int targrows, double *totalrows, + double *totaldeadrows); + + #endif /* VACUUM_H */ diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 4f77692..ae82a1e 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -376,5 +376,13 @@ extern void check_exclusion_constraint(Relation heap, Relation index, Datum *values, bool *isnull, EState *estate, bool newIndex); - +/* + * prototypes from functions in execColumnStore.c + */ +extern void ExecOpenColumnStores(ResultRelInfo *resultRelInfo); +extern void ExecCloseColumnStores(ResultRelInfo *resultRelInfo); +extern void ExecInsertColStoreTuples(HeapTuple tuple, EState *estate); +extern void ExecBatchInsertColStoreTuples(int ntuples, HeapTuple *tuples, + EState *estate); +extern void ExecTruncateColumnStores(Relation rel); #endif /* EXECUTOR_H */ diff --git a/src/include/executor/nodeColumnStorescan.h b/src/include/executor/nodeColumnStorescan.h new file mode 100644 index 0000000..152d939 --- /dev/null +++ b/src/include/executor/nodeColumnStorescan.h @@ -0,0 +1,26 @@ +/*------------------------------------------------------------------------- + * + * nodeColumnStorescan.h + * + * + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/executor/nodeColumnStorescan.h + * + *------------------------------------------------------------------------- + */ +#ifndef NODECOLSTORESCAN_H +#define NODECOLSTORESCAN_H + +#include "nodes/execnodes.h" + +extern ColumnStoreScanState *ExecInitColumnStoreScan(ColumnStoreScan *node, EState *estate, int eflags); +extern TupleTableSlot *ExecColumnStoreScan(ColumnStoreScanState *node); +extern void ExecEndColumnStoreScan(ColumnStoreScanState *node); +extern void ExecColumnStoreScanMarkPos(ColumnStoreScanState *node); +extern void ExecColumnStoreScanRestrPos(ColumnStoreScanState *node); +extern void ExecReScanColumnStoreScan(ColumnStoreScanState *node); + +#endif /* NODECOLSTORESCAN_H */ diff --git a/src/include/executor/tuptable.h b/src/include/executor/tuptable.h index 00686b0..412c8c3 100644 --- a/src/include/executor/tuptable.h +++ b/src/include/executor/tuptable.h @@ -168,7 +168,7 @@ extern TupleTableSlot *ExecMakeSlotContentsReadOnly(TupleTableSlot *slot); /* in access/common/heaptuple.c */ extern Datum slot_getattr(TupleTableSlot *slot, int attnum, bool *isnull); extern void slot_getallattrs(TupleTableSlot *slot); -extern void slot_getsomeattrs(TupleTableSlot *slot, int attnum); +extern void slot_getsomeattrs(TupleTableSlot *slot, int phynum); extern bool slot_attisnull(TupleTableSlot *slot, int attnum); #endif /* TUPTABLE_H */ diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h index 5ccf470..47ccd65 100644 --- a/src/include/nodes/execnodes.h +++ b/src/include/nodes/execnodes.h @@ -77,6 +77,30 @@ typedef struct IndexInfo } IndexInfo; /* ---------------- + * ColumnStoreHandler + * + * This struct holds the information needed to operate a column store + * + * relationDesc the column store relation + * lockmode the lock mode to hold on the relation + * opaque opaque pointer used by the access method + * ColumnStoreRoutine ColumnStore callback functions + * NumColumnStoreAttrs number of columns in this column store + * KeyAttrNumbers underlying-rel attribute numbers used as keys + * ---------------- + */ +typedef struct ColumnStoreHandler +{ + NodeTag type; + Relation csh_relationDesc; + LOCKMODE csh_lockmode; + void *csh_opaque; + struct ColumnStoreRoutine *csh_ColumnStoreRoutine; + int csh_NumColumnStoreAttrs; + AttrNumber csh_KeyAttrNumbers[INDEX_MAX_KEYS]; +} ColumnStoreHandler; + +/* ---------------- * ExprContext_CB * * List of callbacks to be called at ExprContext shutdown. @@ -305,6 +329,7 @@ typedef struct JunkFilter * NumIndices # of indices existing on result relation * IndexRelationDescs array of relation descriptors for indices * IndexRelationInfo array of key/attr info for indices + * ColumnStoreHandler list of ColumnStoreHandler * TrigDesc triggers to be fired, if any * TrigFunctions cached lookup info for trigger functions * TrigWhenExprs array of trigger WHEN expr states @@ -328,6 +353,7 @@ typedef struct ResultRelInfo int ri_NumIndices; RelationPtr ri_IndexRelationDescs; IndexInfo **ri_IndexRelationInfo; + List *ri_ColumnStoreHandler; TriggerDesc *ri_TrigDesc; FmgrInfo *ri_TrigFunctions; List **ri_TrigWhenExprs; @@ -1634,6 +1660,18 @@ typedef struct CustomScanState const CustomExecMethods *methods; } CustomScanState; +/* ---------------- + * ColumnStoreScanState information + * ---------------- + */ +typedef struct ColumnStoreScanState +{ + ScanState ss; /* its first field is NodeTag */ + int eflags; /* capability flags */ + + struct ColumnStoreHandler *csthandler; +} ColumnStoreScanState; + /* ---------------------------------------------------------------- * Join State Information * ---------------------------------------------------------------- diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 603edd3..4989d31 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -37,6 +37,7 @@ typedef enum NodeTag T_ResultRelInfo, T_EState, T_TupleTableSlot, + T_ColumnStoreHandler, /* * TAGS FOR PLAN NODES (plannodes.h) @@ -83,6 +84,7 @@ typedef enum NodeTag T_NestLoopParam, T_PlanRowMark, T_PlanInvalItem, + T_ColumnStoreScan, /* * TAGS FOR PLAN STATE NODES (execnodes.h) @@ -112,6 +114,7 @@ typedef enum NodeTag T_WorkTableScanState, T_ForeignScanState, T_CustomScanState, + T_ColumnStoreScanState, T_JoinState, T_NestLoopState, T_MergeJoinState, @@ -235,6 +238,7 @@ typedef enum NodeTag T_TidPath, T_ForeignPath, T_CustomPath, + T_ColumnStoreScanPath, T_AppendPath, T_MergeAppendPath, T_ResultPath, @@ -248,9 +252,11 @@ typedef enum NodeTag T_PlaceHolderVar, T_SpecialJoinInfo, T_AppendRelInfo, + T_ColstoreRelInfo, T_PlaceHolderInfo, T_MinMaxAggInfo, T_PlannerParamItem, + T_ColumnStoreOptInfo, /* * TAGS FOR MEMORY NODES (memnodes.h) @@ -381,6 +387,7 @@ typedef enum NodeTag T_CreatePolicyStmt, T_AlterPolicyStmt, T_CreateTransformStmt, + T_CreateColumnStoreAMStmt, /* * TAGS FOR PARSE TREE NODES (parsenodes.h) @@ -398,6 +405,7 @@ typedef enum NodeTag T_MultiAssignRef, T_TypeCast, T_CollateClause, + T_ColumnStoreClause, T_SortBy, T_WindowDef, T_RangeSubselect, @@ -454,7 +462,8 @@ typedef enum NodeTag T_TIDBitmap, /* in nodes/tidbitmap.h */ T_InlineCodeBlock, /* in nodes/parsenodes.h */ T_FdwRoutine, /* in foreign/fdwapi.h */ - T_TsmRoutine /* in access/tsmapi.h */ + T_TsmRoutine, /* in access/tsmapi.h */ + T_ColumnStoreRoutine /* in colstore/colstoreapi.h */ } NodeTag; /* diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index abd4dd1..50b3f42 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -565,6 +565,24 @@ typedef struct RangeTableSample } RangeTableSample; /* + * ColumnStoreClause - a COLUMN STORE declaration + * + * When this appears as a column constraint, the store includes only that + * column, and the column list is NIL. When it appears as a table constraint, + * the column list contains one or more column names. + */ +typedef struct ColumnStoreClause +{ + NodeTag type; + char *name; /* column store name */ + char *storetype; /* column store type */ + List *columns; /* list of String column names, or NIL */ + List *options; /* list of DefElem options */ + char *tablespacename; /* table space to use, or NULL */ + int location; +} ColumnStoreClause; + +/* * ColumnDef - column definition (used in various creates) * * If the column has a default value, we may have the value expression @@ -577,6 +595,10 @@ typedef struct RangeTableSample * (represented as a CollateClause with arg==NULL) or cooked form * (the collation's OID). * + * Similarly, a COLUMN STORE specification may come in raw form (a + * ColumnStoreClause) or cooked (the OID of the corresponding store in the + * parent relation. Note we only reuse its name, not the store itself.) + * * The constraints list may contain a CONSTR_DEFAULT item in a raw * parsetree produced by gram.y, but transformCreateStmt will remove * the item and set raw_default instead. CONSTR_DEFAULT items @@ -596,6 +618,7 @@ typedef struct ColumnDef Node *cooked_default; /* default value (transformed expr tree) */ CollateClause *collClause; /* untransformed COLLATE spec, if any */ Oid collOid; /* collation OID (InvalidOid if not set) */ + ColumnStoreClause *cstoreClause; /* untransformed COL STORE, if any */ List *constraints; /* other constraints on column */ List *fdwoptions; /* per-column FDW options */ int location; /* parse location, or -1 if none/unknown */ @@ -801,6 +824,7 @@ typedef struct RangeTblEntry */ Oid relid; /* OID of the relation */ char relkind; /* relation kind (see pg_class.relkind) */ + bool relhascstore; /* whether column stores are present */ struct TableSampleClause *tablesample; /* sampling info, or NULL */ /* @@ -1386,6 +1410,7 @@ typedef enum ObjectType OBJECT_CAST, OBJECT_COLUMN, OBJECT_COLLATION, + OBJECT_COLSTORE, OBJECT_CONVERSION, OBJECT_DATABASE, OBJECT_DEFAULT, @@ -1734,11 +1759,13 @@ typedef struct VariableShowStmt /* ---------------------- * Create Table Statement * - * NOTE: in the raw gram.y output, ColumnDef and Constraint nodes are - * intermixed in tableElts, and constraints is NIL. After parse analysis, - * tableElts contains just ColumnDefs, and constraints contains just - * Constraint nodes (in fact, only CONSTR_CHECK nodes, in the present - * implementation). + * NOTE: in the raw gram.y output, ColumnDef, ColumnStoreClause and Constraint + * nodes are intermixed in tableElts, and constraints and colstores are NIL. + * After parse analysis, tableElts contains just ColumnDefs, constraints + * contains just Constraint nodes (in fact, only CONSTR_CHECK nodes, in the + * present implementation), and colstores contains only ColumnStoreClause + * nodes (further note that ColumnDef nodes might have ColumnStoreClause nodes + * within. Those are left alone during parse analysis.) * ---------------------- */ @@ -1751,6 +1778,7 @@ typedef struct CreateStmt * inhRelation) */ TypeName *ofTypename; /* OF typename */ List *constraints; /* constraints (list of Constraint nodes) */ + List *colstores; /* list of ColumnStoreClause nodes */ List *options; /* options from WITH clause */ OnCommitAction oncommit; /* what do we do at COMMIT? */ char *tablespacename; /* table space to use, or NULL */ @@ -2041,6 +2069,19 @@ typedef struct ImportForeignSchemaStmt List *options; /* list of options to pass to FDW */ } ImportForeignSchemaStmt; + +/* ---------------------- + * Create COLUMN STORE ACCESS METHOD Statements + * ---------------------- + */ + +typedef struct CreateColumnStoreAMStmt +{ + NodeTag type; + char *cstamname; /* column store AM name */ + List *func_options; /* HANDLER option */ +} CreateColumnStoreAMStmt; + /*---------------------- * Create POLICY Statement *---------------------- diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index 37086c6..10e0a1f 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -577,6 +577,14 @@ typedef struct CustomScan const CustomScanMethods *methods; } CustomScan; +/* ---------------- + * column store scan node + * ---------------- + */ +typedef struct ColumnStoreScan +{ + Scan scan; +} ColumnStoreScan; /* * ========== * Join nodes diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h index 5393005..d1b0a5f 100644 --- a/src/include/nodes/relation.h +++ b/src/include/nodes/relation.h @@ -228,6 +228,8 @@ typedef struct PlannerInfo List *append_rel_list; /* list of AppendRelInfos */ + List *colstore_rel_list; /* list of ColstoreRelInfo */ + List *rowMarks; /* list of PlanRowMarks */ List *placeholder_list; /* list of PlaceHolderInfos */ @@ -379,6 +381,7 @@ typedef struct PlannerInfo * pages - number of disk pages in relation (zero if not a table) * tuples - number of tuples in relation (not considering restrictions) * allvisfrac - fraction of disk pages that are marked all-visible + * cstore - pointer to ColumnStoreOptInfo (NULL if not a column store) * subplan - plan for subquery (NULL if it's not a subquery) * subroot - PlannerInfo for subquery (NULL if it's not a subquery) * subplan_params - list of PlannerParamItems to be passed to subquery @@ -479,9 +482,11 @@ typedef struct RelOptInfo List *lateral_vars; /* LATERAL Vars and PHVs referenced by rel */ Relids lateral_referencers; /* rels that reference me laterally */ List *indexlist; /* list of IndexOptInfo */ + List *cstlist; /* list of ColumnStoreOptInfo (if relation) */ BlockNumber pages; /* size estimates derived from pg_class */ double tuples; double allvisfrac; + struct ColumnStoreOptInfo *cstore; /* if column store */ /* use "struct Plan" to avoid including plannodes.h here */ struct Plan *subplan; /* if subquery */ PlannerInfo *subroot; /* if subquery */ @@ -573,6 +578,35 @@ typedef struct IndexOptInfo bool amhasgetbitmap; /* does AM have amgetbitmap interface? */ } IndexOptInfo; +/* + * ColumnStoreOptInfo + * Per-colstore information for planning/optimization + * + * cstkeys[] has ncolumns entries, we don't allow expression in colstores + * + * TODO maybe should have a bunch of capability flags like IndexOptInfo + */ +typedef struct ColumnStoreOptInfo +{ + NodeTag type; + + Oid colstoreoid; /* OID of the column store relation */ + Oid reltablespace; /* tablespace of colstore (not table) */ + RelOptInfo *rel; /* back-link to colstore's table */ + + /* colstore-size statistics (from pg_class and elsewhere) */ + BlockNumber pages; /* number of disk pages in colstore */ + double tuples; /* number of index tuples in colstore */ + + /* colstore descriptor information (from pg_cstore) */ + int ncolumns; /* number of columns in index */ + int *cstkeys; /* column numbers of colstore's keys */ + Oid cstam; /* OID of the access method (in pg_cstore_am) */ + + RegProcedure cstcostestimate; /* OID of the colstore method's cost fcn */ + + List *csttlist; /* targetlist representing colstore's columns */ +} ColumnStoreOptInfo; /* * EquivalenceClasses @@ -966,6 +1000,15 @@ typedef struct CustomPath } CustomPath; /* + * ColumnStoreScanPath represents scan of column store. + */ +typedef struct ColumnStoreScanPath +{ + Path path; /* this path node */ + ColumnStoreOptInfo *colstore; /* column store info */ +} ColumnStoreScanPath; + +/* * AppendPath represents an Append plan, ie, successive execution of * several member plans. * @@ -1543,6 +1586,16 @@ typedef struct AppendRelInfo Oid parent_reloid; /* OID of parent relation */ } AppendRelInfo; +/* for column stores. XXX Improve this comment */ +typedef struct ColstoreRelInfo +{ + NodeTag type; + + Index parent_relid; /* parent RT index */ + Index child_relid; /* colstore RT index */ + Oid child_oid; /* colstore's OID */ +} ColstoreRelInfo; + /* * For each distinct placeholder expression generated during planning, we * store a PlaceHolderInfo node in the PlannerInfo node's placeholder_list. diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index ac21a3a..d6f027f 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -75,6 +75,8 @@ extern void cost_seqscan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info, int nworkers); extern void cost_samplescan(Path *path, PlannerInfo *root, RelOptInfo *baserel, ParamPathInfo *param_info); +extern void cost_colstore_scan(Path *path, PlannerInfo *root, + RelOptInfo *baserel, ParamPathInfo *param_info); extern void cost_index(IndexPath *path, PlannerInfo *root, double loop_count); extern void cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h index 8fb9eda..8b3cbc1 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -88,6 +88,7 @@ extern ForeignPath *create_foreignscan_path(PlannerInfo *root, RelOptInfo *rel, Relids required_outer, Path *fdw_outerpath, List *fdw_private); +extern Path *create_colstore_scan_path(PlannerInfo *root, RelOptInfo *rel); extern Relids calc_nestloop_required_outer(Path *outer_path, Path *inner_path); extern Relids calc_non_nestloop_required_outer(Path *outer_path, Path *inner_path); diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h index f96e9ee..0638832 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -98,6 +98,7 @@ extern int from_collapse_limit; extern int join_collapse_limit; extern void add_base_rels_to_query(PlannerInfo *root, Node *jtnode); +extern Node *expand_column_store_relations(PlannerInfo *root, Node *jtnode); extern void build_base_rel_tlists(PlannerInfo *root, List *final_tlist); extern void add_vars_to_targetlist(PlannerInfo *root, List *vars, Relids where_needed, bool create_new_ph); diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 812ca83..3ecd40e 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -239,6 +239,7 @@ PG_KEYWORD("mapping", MAPPING, UNRESERVED_KEYWORD) PG_KEYWORD("match", MATCH, UNRESERVED_KEYWORD) PG_KEYWORD("materialized", MATERIALIZED, UNRESERVED_KEYWORD) PG_KEYWORD("maxvalue", MAXVALUE, UNRESERVED_KEYWORD) +PG_KEYWORD("method", METHOD, UNRESERVED_KEYWORD) PG_KEYWORD("minute", MINUTE_P, UNRESERVED_KEYWORD) PG_KEYWORD("minvalue", MINVALUE, UNRESERVED_KEYWORD) PG_KEYWORD("mode", MODE, UNRESERVED_KEYWORD) @@ -365,6 +366,7 @@ PG_KEYWORD("statistics", STATISTICS, UNRESERVED_KEYWORD) PG_KEYWORD("stdin", STDIN, UNRESERVED_KEYWORD) PG_KEYWORD("stdout", STDOUT, UNRESERVED_KEYWORD) PG_KEYWORD("storage", STORAGE, UNRESERVED_KEYWORD) +PG_KEYWORD("store", STORE, UNRESERVED_KEYWORD) PG_KEYWORD("strict", STRICT_P, UNRESERVED_KEYWORD) PG_KEYWORD("strip", STRIP_P, UNRESERVED_KEYWORD) PG_KEYWORD("substring", SUBSTRING, COL_NAME_KEYWORD) diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index e610bf3..5a6f5d3 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -464,6 +464,7 @@ extern Datum pg_size_pretty(PG_FUNCTION_ARGS); extern Datum pg_size_pretty_numeric(PG_FUNCTION_ARGS); extern Datum pg_table_size(PG_FUNCTION_ARGS); extern Datum pg_indexes_size(PG_FUNCTION_ARGS); +extern Datum pg_column_stores_size(PG_FUNCTION_ARGS); extern Datum pg_relation_filenode(PG_FUNCTION_ARGS); extern Datum pg_filenode_relation(PG_FUNCTION_ARGS); extern Datum pg_relation_filepath(PG_FUNCTION_ARGS); @@ -568,6 +569,8 @@ extern Datum fdw_handler_in(PG_FUNCTION_ARGS); extern Datum fdw_handler_out(PG_FUNCTION_ARGS); extern Datum tsm_handler_in(PG_FUNCTION_ARGS); extern Datum tsm_handler_out(PG_FUNCTION_ARGS); +extern Datum cstore_handler_in(PG_FUNCTION_ARGS); +extern Datum cstore_handler_out(PG_FUNCTION_ARGS); extern Datum internal_in(PG_FUNCTION_ARGS); extern Datum internal_out(PG_FUNCTION_ARGS); extern Datum opaque_in(PG_FUNCTION_ARGS); @@ -1246,6 +1249,9 @@ extern Datum pg_identify_object_as_address(PG_FUNCTION_ARGS); /* catalog/objectaddress.c */ extern Datum pg_get_object_address(PG_FUNCTION_ARGS); +/* colstore/ */ +extern Datum vertical_cstore_handler(PG_FUNCTION_ARGS); + /* commands/constraint.c */ extern Datum unique_key_recheck(PG_FUNCTION_ARGS); diff --git a/src/include/utils/lsyscache.h b/src/include/utils/lsyscache.h index dcc421f..c8eb5f7 100644 --- a/src/include/utils/lsyscache.h +++ b/src/include/utils/lsyscache.h @@ -99,6 +99,7 @@ extern float4 get_func_cost(Oid funcid); extern float4 get_func_rows(Oid funcid); extern Oid get_relname_relid(const char *relname, Oid relnamespace); extern char *get_rel_name(Oid relid); +extern char *get_cstore_name(Oid relid); extern Oid get_rel_namespace(Oid relid); extern Oid get_rel_type_id(Oid relid); extern char get_rel_relkind(Oid relid); diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 8a55a09..29a24b0 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -17,6 +17,7 @@ #include "access/tupdesc.h" #include "catalog/pg_am.h" #include "catalog/pg_class.h" +#include "catalog/pg_cstore.h" #include "catalog/pg_index.h" #include "fmgr.h" #include "nodes/bitmapset.h" @@ -79,6 +80,7 @@ typedef struct RelationData bool rd_isvalid; /* relcache entry is valid */ char rd_indexvalid; /* state of rd_indexlist: 0 = not valid, 1 = * valid, 2 = temporarily forced */ + bool rd_cstvalid; /* rd_cstlist is valid */ /* * rd_createSubid is the ID of the highest subtransaction the rel has @@ -117,6 +119,9 @@ typedef struct RelationData Bitmapset *rd_keyattr; /* cols that can be ref'd by foreign keys */ Bitmapset *rd_idattr; /* included in replica identity index */ + /* data managed by RelationGetColStoreList: */ + List *rd_cstlist; /* list of OIDs of colstores on relation */ + /* * rd_options is set whenever rd_rel is loaded into the relcache entry. * Note that you can NOT look into rd_rel for this data. NULL means "use @@ -171,6 +176,14 @@ typedef struct RelationData /* use "struct" here to avoid needing to include fdwapi.h: */ struct FdwRoutine *rd_fdwroutine; /* cached function pointers, or NULL */ + /* These are non-NULL only for a column store relation: */ + Form_pg_cstore rd_cstore; /* pg_cstore tuple describing this colstore */ + /* use "struct" here to avoid needing to include htup.h: */ + struct HeapTupleData *rd_cstoretuple; /* all of pg_cstore tuple */ + + /* use "struct" here to avoid needing to include colstoreapi.h: */ + struct ColumnStoreRoutine *rd_colstoreroutine; /* cached function pointers, or NULL */ + /* * Hack for CLUSTER, rewriting ALTER TABLE, etc: when writing a new * version of a table, we need to make any toast pointers inserted into it diff --git a/src/include/utils/relcache.h b/src/include/utils/relcache.h index 6953281..0a8d52c 100644 --- a/src/include/utils/relcache.h +++ b/src/include/utils/relcache.h @@ -38,6 +38,7 @@ extern void RelationClose(Relation relation); * Routines to compute/retrieve additional cached information */ extern List *RelationGetIndexList(Relation relation); +extern List *RelationGetColStoreList(Relation relation); extern Oid RelationGetOidIndex(Relation relation); extern Oid RelationGetReplicaIndex(Relation relation); extern List *RelationGetIndexExpressions(Relation relation); diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index 18404e2..abaeb21 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -52,6 +52,9 @@ enum SysCacheIdentifier CONNAMENSP, CONSTROID, CONVOID, + CSTOREAMNAME, + CSTOREAMOID, + CSTOREOID, DATABASEOID, DEFACLROLENSPOBJ, ENUMOID, diff --git a/src/test/regress/expected/cstore.out b/src/test/regress/expected/cstore.out new file mode 100644 index 0000000..e6c6ac4 --- /dev/null +++ b/src/test/regress/expected/cstore.out @@ -0,0 +1,376 @@ +CREATE COLUMN STORE ACCESS METHOD store_am_one HANDLER vertical_cstore_handler; +CREATE COLUMN STORE ACCESS METHOD store_am_two HANDLER vertical_cstore_handler; +-- column-level column store definition +-- missing USING clause +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE foo, + c INT +); +ERROR: syntax error at or near "," +LINE 3: b INT COLUMN STORE foo, + ^ +-- missing column store name +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE USING store_am_one, + c INT +); +ERROR: syntax error at or near "USING" +LINE 3: b INT COLUMN STORE USING store_am_one, + ^ +-- missing USING and column store name +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE, + c INT +); +ERROR: syntax error at or near "," +LINE 3: b INT COLUMN STORE, + ^ +-- missing column store name +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE USING store_am_one, + c INT +); +ERROR: syntax error at or near "USING" +LINE 3: b INT COLUMN STORE USING store_am_one, + ^ +-- unknown name of a column store AM +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE foo USING no_store_am, + c INT +); +ERROR: column store access method "no_store_am" does not exist +-- conflicting column store name +CREATE TABLE test_columnar_single_conflict ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT COLUMN STORE foo USING store_am_one, + d INT +); +ERROR: duplicate column store name "foo" +-- correct definition (single store) +CREATE TABLE test_columnar_single_ok ( + a INT, + b INT COLUMN STORE foo1 USING store_am_one, + c INT +); +\d test_columnar_single_ok +Table "public.test_columnar_single_ok" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | +Column stores: + "foo1" USING store_am_one (b) + +-- correct definition (two stores) +CREATE TABLE test_columnar_single_ok2 ( + a INT, + b INT COLUMN STORE foo1 USING store_am_one, + c INT COLUMN STORE foo2 USING store_am_two, + d INT +); +\d test_columnar_single_ok2 +Table "public.test_columnar_single_ok2" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | + d | integer | +Column stores: + "foo1" USING store_am_one (b) + "foo2" USING store_am_two (c) + +-- table-level column store definition +-- no column list +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one +); +ERROR: syntax error at or near ")" +LINE 7: ); + ^ +-- empty column list +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one () +); +ERROR: syntax error at or near ")" +LINE 6: COLUMN STORE foo USING store_am_one () + ^ +-- invalid column in store +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one (z) +); +ERROR: no column "z" in the table +-- unknown name of a column store AM +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING no_store_am (b,c) +); +ERROR: column store access method "no_store_am" does not exist +-- conflicting column store name +CREATE TABLE test_columnar_multi_conflict ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one (a,b), + COLUMN STORE foo USING store_am_one (c,d) +); +ERROR: duplicate column store name "foo" +-- overlapping list of columns +CREATE TABLE test_columnar_multi_conflict2 ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo1 USING store_am_one (a,b), + COLUMN STORE foo2 USING store_am_two (b,c,d) +); +ERROR: column already in a store +-- correct definition (single store) +CREATE TABLE test_columnar_multi_ok ( + a INT, + b INT, + c INT, + COLUMN STORE foo USING store_am_one (a,b) +); +\d test_columnar_multi_ok +Table "public.test_columnar_multi_ok" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | +Column stores: + "foo" USING store_am_one (a, b) + +-- correct definition (two stores) +CREATE TABLE test_columnar_multi_ok2 ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo1 USING store_am_one (a,b), + COLUMN STORE foo2 USING store_am_one (c,d) +); +\d test_columnar_multi_ok2 +Table "public.test_columnar_multi_ok2" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | + d | integer | +Column stores: + "foo1" USING store_am_one (a, b) + "foo2" USING store_am_one (c, d) + +-- combination of column-level and table-level column stores +-- conflicting column store name +CREATE TABLE test_columnar_multi_conflict ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT, + d INT, + COLUMN STORE foo USING store_am_one (c,d) +); +ERROR: duplicate column store name "foo" +-- overlapping list of columns +CREATE TABLE test_columnar_combi_conflict2 ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT, + d INT, + COLUMN STORE foo2 USING store_am_two (b,c,d) +); +ERROR: column already in a store +-- correct definition (two stores) +CREATE TABLE test_columnar_combi_ok ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT, + d INT, + COLUMN STORE foo2 USING store_am_one (c,d) +); +\d test_columnar_combi_ok +Table "public.test_columnar_combi_ok" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | + d | integer | +Column stores: + "foo" USING store_am_one (b) + "foo2" USING store_am_one (c, d) + +-- test cleanup +CREATE TABLE cstore_oids AS +SELECT cststoreid + FROM pg_cstore JOIN pg_class ON (pg_cstore.cstrelid = pg_class.oid) + WHERE relname IN ('test_columnar_single_ok', + 'test_columnar_single_ok2', + 'test_columnar_multi_ok', + 'test_columnar_multi_ok2', + 'test_columnar_combi_ok'); +CREATE TABLE cstore_oids_2 AS +SELECT pg_class.oid + FROM pg_class JOIN cstore_oids ON (pg_class.oid = cstore_oids.cststoreid); +DROP TABLE test_columnar_single_ok; +DROP TABLE test_columnar_single_ok2; +DROP TABLE test_columnar_multi_ok; +DROP TABLE test_columnar_multi_ok2; +DROP TABLE test_columnar_combi_ok; +-- should return 0 +SELECT COUNT(*) FROM pg_class WHERE oid IN (SELECT cststoreid FROM cstore_oids); + count +------- + 0 +(1 row) + +SELECT COUNT(*) FROM pg_class WHERE oid IN (SELECT oid FROM cstore_oids_2); + count +------- + 0 +(1 row) + +SELECT COUNT(*) FROM pg_cstore WHERE cststoreid IN (SELECT oid FROM cstore_oids); + count +------- + 0 +(1 row) + +SELECT COUNT(*) FROM pg_attribute WHERE attrelid IN (SELECT cststoreid FROM cstore_oids); + count +------- + 0 +(1 row) + +SELECT COUNT(*) FROM pg_attribute WHERE attrelid IN (SELECT oid FROM cstore_oids_2); + count +------- + 0 +(1 row) + +DROP TABLE cstore_oids; +DROP TABLE cstore_oids_2; +-- INHERITANCE +-- parent table with two column stores +CREATE TABLE parent_table ( + a INT, + b INT COLUMN STORE foo1 USING store_am_one, + c INT, + d INT, + e INT, + COLUMN STORE foo2 USING store_am_two (d,e) +); +-- child table with two separate column stores +CREATE TABLE child_table_1 ( + f INT, + g INT COLUMN STORE foo1c USING store_am_one, + h INT, + i INT, + COLUMN STORE foo2c USING store_am_two(h,i) +) INHERITS (parent_table); +\d child_table_1 + Table "public.child_table_1" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | + d | integer | + e | integer | + f | integer | + g | integer | + h | integer | + i | integer | +Inherits: parent_table +Column stores: + "foo1" USING store_am_one (b) + "foo2" USING store_am_two (d, e) + "foo1c" USING store_am_one (g) + "foo2c" USING store_am_two (h, i) + +-- child table with two column stores - one modifying, one redefining the parent +CREATE TABLE child_table_2 ( + f INT, + g INT COLUMN STORE foo1c USING store_am_one, -- new column store + h INT, + i INT, + COLUMN STORE foo2c USING store_am_two(b,h,i) -- redefines the parent colstore +) INHERITS (parent_table); +\d child_table_2 + Table "public.child_table_2" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | + d | integer | + e | integer | + f | integer | + g | integer | + h | integer | + i | integer | +Inherits: parent_table +Column stores: + "foo2c" USING store_am_two (b, h, i) + "foo2" USING store_am_two (d, e) + "foo1c" USING store_am_one (g) + +-- child table with a single column store of the whole table +CREATE TABLE child_table_3 ( + f INT, + g INT, + h INT, + i INT, + COLUMN STORE foo1 USING store_am_one(a,b,c,d,e,f,g,h,i) +) INHERITS (parent_table); +\d child_table_3 + Table "public.child_table_3" + Column | Type | Modifiers +--------+---------+----------- + a | integer | + b | integer | + c | integer | + d | integer | + e | integer | + f | integer | + g | integer | + h | integer | + i | integer | +Inherits: parent_table +Column stores: + "foo1" USING store_am_one (a, b, c, d, e, f, g, h, i) + +--- FIXME -- add tests with multiple inheritance +DROP TABLE parent_table CASCADE; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table child_table_1 +drop cascades to table child_table_2 +drop cascades to table child_table_3 +--- delete the fake cstore AM records +-- FIXME -- this should be a DROP command +DELETE FROM pg_cstore_am WHERE cstamname IN ('store_am_one', 'store_am_two'); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 80374e4..413f630 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1359,6 +1359,7 @@ pg_matviews| SELECT n.nspname AS schemaname, pg_get_userbyid(c.relowner) AS matviewowner, t.spcname AS tablespace, c.relhasindex AS hasindexes, + c.relhascstore AS hascstores, c.relispopulated AS ispopulated, pg_get_viewdef(c.oid) AS definition FROM ((pg_class c @@ -2068,6 +2069,7 @@ pg_tables| SELECT n.nspname AS schemaname, pg_get_userbyid(c.relowner) AS tableowner, t.spcname AS tablespace, c.relhasindex AS hasindexes, + c.relhascstore AS hascstores, c.relhasrules AS hasrules, c.relhastriggers AS hastriggers, c.relrowsecurity AS rowsecurity diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index eb0bc88..6abdab7 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -7,170 +7,172 @@ VACUUM; -- -- temporarily disable fancy output, so catalog changes create less diff noise \a\t -SELECT relname, relhasindex +SELECT relname, relhasindex, relhascstore FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = relnamespace WHERE relkind = 'r' AND (nspname ~ '^pg_temp_') IS NOT TRUE ORDER BY relname; -a|f -a_star|f -abstime_tbl|f -aggtest|f -array_index_op_test|t -array_op_test|f -b|f -b_star|f -box_tbl|f -bprime|f -bt_f8_heap|t -bt_i4_heap|t -bt_name_heap|t -bt_txt_heap|t -c|f -c_star|f -char_tbl|f -check2_tbl|f -check_tbl|f -circle_tbl|t -city|f -copy_tbl|f -d|f -d_star|f -date_tbl|f -default_tbl|f -defaultexpr_tbl|f -dept|f -dupindexcols|t -e_star|f -emp|f -equipment_r|f -f_star|f -fast_emp4000|t -float4_tbl|f -float8_tbl|f -func_index_heap|t -hash_f8_heap|t -hash_i4_heap|t -hash_name_heap|t -hash_txt_heap|t -hobbies_r|f -ihighway|t -inet_tbl|f -inhf|f -inhx|t -insert_tbl|f -int2_tbl|f -int4_tbl|f -int8_tbl|f -interval_tbl|f -iportaltest|f -kd_point_tbl|t -line_tbl|f -log_table|f -lseg_tbl|f -main_table|f -money_data|f -num_data|f -num_exp_add|t -num_exp_div|t -num_exp_ln|t -num_exp_log10|t -num_exp_mul|t -num_exp_power_10_ln|t -num_exp_sqrt|t -num_exp_sub|t -num_input_test|f -num_result|f -onek|t -onek2|t -path_tbl|f -person|f -pg_aggregate|t -pg_am|t -pg_amop|t -pg_amproc|t -pg_attrdef|t -pg_attribute|t -pg_auth_members|t -pg_authid|t -pg_cast|t -pg_class|t -pg_collation|t -pg_constraint|t -pg_conversion|t -pg_database|t -pg_db_role_setting|t -pg_default_acl|t -pg_depend|t -pg_description|t -pg_enum|t -pg_event_trigger|t -pg_extension|t -pg_foreign_data_wrapper|t -pg_foreign_server|t -pg_foreign_table|t -pg_index|t -pg_inherits|t -pg_language|t -pg_largeobject|t -pg_largeobject_metadata|t -pg_namespace|t -pg_opclass|t -pg_operator|t -pg_opfamily|t -pg_pltemplate|t -pg_policy|t -pg_proc|t -pg_range|t -pg_replication_origin|t -pg_rewrite|t -pg_seclabel|t -pg_shdepend|t -pg_shdescription|t -pg_shseclabel|t -pg_statistic|t -pg_tablespace|t -pg_transform|t -pg_trigger|t -pg_ts_config|t -pg_ts_config_map|t -pg_ts_dict|t -pg_ts_parser|t -pg_ts_template|t -pg_type|t -pg_user_mapping|t -point_tbl|t -polygon_tbl|t -quad_point_tbl|t -radix_text_tbl|t -ramp|f -real_city|f -reltime_tbl|f -road|t -shighway|t -slow_emp4000|f -sql_features|f -sql_implementation_info|f -sql_languages|f -sql_packages|f -sql_parts|f -sql_sizing|f -sql_sizing_profiles|f -stud_emp|f -student|f -tenk1|t -tenk2|t -test_range_excl|t -test_range_gist|t -test_range_spgist|t -test_tsvector|f -testjsonb|f -text_tbl|f -time_tbl|f -timestamp_tbl|f -timestamptz_tbl|f -timetz_tbl|f -tinterval_tbl|f -varchar_tbl|f +a|f|f +a_star|f|f +abstime_tbl|f|f +aggtest|f|f +array_index_op_test|t|f +array_op_test|f|f +b|f|f +b_star|f|f +box_tbl|f|f +bprime|f|f +bt_f8_heap|t|f +bt_i4_heap|t|f +bt_name_heap|t|f +bt_txt_heap|t|f +c|f|f +c_star|f|f +char_tbl|f|f +check2_tbl|f|f +check_tbl|f|f +circle_tbl|t|f +city|f|f +copy_tbl|f|f +d|f|f +d_star|f|f +date_tbl|f|f +default_tbl|f|f +defaultexpr_tbl|f|f +dept|f|f +dupindexcols|t|f +e_star|f|f +emp|f|f +equipment_r|f|f +f_star|f|f +fast_emp4000|t|f +float4_tbl|f|f +float8_tbl|f|f +func_index_heap|t|f +hash_f8_heap|t|f +hash_i4_heap|t|f +hash_name_heap|t|f +hash_txt_heap|t|f +hobbies_r|f|f +ihighway|t|f +inet_tbl|f|f +inhf|f|f +inhx|t|f +insert_tbl|f|f +int2_tbl|f|f +int4_tbl|f|f +int8_tbl|f|f +interval_tbl|f|f +iportaltest|f|f +kd_point_tbl|t|f +line_tbl|f|f +log_table|f|f +lseg_tbl|f|f +main_table|f|f +money_data|f|f +num_data|f|f +num_exp_add|t|f +num_exp_div|t|f +num_exp_ln|t|f +num_exp_log10|t|f +num_exp_mul|t|f +num_exp_power_10_ln|t|f +num_exp_sqrt|t|f +num_exp_sub|t|f +num_input_test|f|f +num_result|f|f +onek|t|f +onek2|t|f +path_tbl|f|f +person|f|f +pg_aggregate|t|f +pg_am|t|f +pg_amop|t|f +pg_amproc|t|f +pg_attrdef|t|f +pg_attribute|t|f +pg_auth_members|t|f +pg_authid|t|f +pg_cast|t|f +pg_class|t|f +pg_collation|t|f +pg_constraint|t|f +pg_conversion|t|f +pg_cstore|t|f +pg_cstore_am|t|f +pg_database|t|f +pg_db_role_setting|t|f +pg_default_acl|t|f +pg_depend|t|f +pg_description|t|f +pg_enum|t|f +pg_event_trigger|t|f +pg_extension|t|f +pg_foreign_data_wrapper|t|f +pg_foreign_server|t|f +pg_foreign_table|t|f +pg_index|t|f +pg_inherits|t|f +pg_language|t|f +pg_largeobject|t|f +pg_largeobject_metadata|t|f +pg_namespace|t|f +pg_opclass|t|f +pg_operator|t|f +pg_opfamily|t|f +pg_pltemplate|t|f +pg_policy|t|f +pg_proc|t|f +pg_range|t|f +pg_replication_origin|t|f +pg_rewrite|t|f +pg_seclabel|t|f +pg_shdepend|t|f +pg_shdescription|t|f +pg_shseclabel|t|f +pg_statistic|t|f +pg_tablespace|t|f +pg_transform|t|f +pg_trigger|t|f +pg_ts_config|t|f +pg_ts_config_map|t|f +pg_ts_dict|t|f +pg_ts_parser|t|f +pg_ts_template|t|f +pg_type|t|f +pg_user_mapping|t|f +point_tbl|t|f +polygon_tbl|t|f +quad_point_tbl|t|f +radix_text_tbl|t|f +ramp|f|f +real_city|f|f +reltime_tbl|f|f +road|t|f +shighway|t|f +slow_emp4000|f|f +sql_features|f|f +sql_implementation_info|f|f +sql_languages|f|f +sql_packages|f|f +sql_parts|f|f +sql_sizing|f|f +sql_sizing_profiles|f|f +stud_emp|f|f +student|f|f +tenk1|t|f +tenk2|t|f +test_range_excl|t|f +test_range_gist|t|f +test_range_spgist|t|f +test_tsvector|f|f +testjsonb|f|f +text_tbl|f|f +time_tbl|f|f +timestamp_tbl|f|f +timestamptz_tbl|f|f +timetz_tbl|f|f +tinterval_tbl|f|f +varchar_tbl|f|f -- restore normal output mode \a\t -- diff --git a/src/test/regress/input/colstore_behave_control.source b/src/test/regress/input/colstore_behave_control.source new file mode 100644 index 0000000..9fcceb1 --- /dev/null +++ b/src/test/regress/input/colstore_behave_control.source @@ -0,0 +1,14 @@ +CREATE TABLE colstore_behave_test +(col1 bigint +,col2 text +,col3 bigint +,col4 integer +,col5 smallint +,col6 text +,col7 char(10) not null +,col8 text +); + +\i @abs_srcdir@/sql/colstore_behave.sql + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/input/colstore_behave_test1.source b/src/test/regress/input/colstore_behave_test1.source new file mode 100644 index 0000000..5230fd8 --- /dev/null +++ b/src/test/regress/input/colstore_behave_test1.source @@ -0,0 +1,15 @@ +CREATE COLUMN STORE ACCESS METHOD test HANDLER vertical_cstore_handler; +CREATE TABLE colstore_behave_test +(col1 bigint +,col2 text +,col3 bigint COLUMN STORE col3 USING test +,col4 integer +,col5 smallint +,col6 text +,col7 char(10) not null +,col8 text +); + +\i @abs_srcdir@/sql/colstore_behave.sql + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/input/colstore_behave_test2.source b/src/test/regress/input/colstore_behave_test2.source new file mode 100644 index 0000000..2c4fe21 --- /dev/null +++ b/src/test/regress/input/colstore_behave_test2.source @@ -0,0 +1,14 @@ +CREATE TABLE colstore_behave_test +(col1 bigint +,col2 text +,col3 bigint +,col4 integer +,col5 smallint +,col6 text COLUMN STORE col6 USING test +,col7 char(10) not null COLUMN STORE col7 USING test +,col8 text COLUMN STORE col8 USING test +); + +\i @abs_srcdir@/sql/colstore_behave.sql + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/input/colstore_behave_test3.source b/src/test/regress/input/colstore_behave_test3.source new file mode 100644 index 0000000..2d364cd --- /dev/null +++ b/src/test/regress/input/colstore_behave_test3.source @@ -0,0 +1,15 @@ +CREATE TABLE colstore_behave_test +(col1 bigint COLUMN STORE col1 USING test +,col2 text +,col3 bigint +,col4 integer +,col5 smallint +,col6 text +,col7 char(10) not null +,col8 text +,COLUMN STORE int USING test (col3, col4, col5) +); + +\i @abs_srcdir@/sql/colstore_behave.sql + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/output/colstore_behave_control.source b/src/test/regress/output/colstore_behave_control.source new file mode 100644 index 0000000..7ec6eba --- /dev/null +++ b/src/test/regress/output/colstore_behave_control.source @@ -0,0 +1,252 @@ +CREATE TABLE colstore_behave_test +(col1 bigint +,col2 text +,col3 bigint +,col4 integer +,col5 smallint +,col6 text +,col7 char(10) not null +,col8 text +); +\i @abs_srcdir@/sql/colstore_behave.sql +-- Required behavior for all column stores +INSERT INTO colstore_behave_test +VALUES (1, 'a', 12, 52, 42, 'col6a', 'col7z', 'col8_1234567'); +INSERT INTO colstore_behave_test +VALUES (2, 'b', 22, 62, 82, 'col6b', 'col7y', 'col8_89101112'); +INSERT INTO colstore_behave_test +VALUES (3, 'c', 32, 72, 92, 'col6c', 'col7x', 'col8_13141516888'); +SELECT col1 FROM colstore_behave_test; + col1 +------ + 1 + 2 + 3 +(3 rows) + +SELECT col2 FROM colstore_behave_test; + col2 +------ + a + b + c +(3 rows) + +SELECT col3 FROM colstore_behave_test; + col3 +------ + 12 + 22 + 32 +(3 rows) + +SELECT col4 FROM colstore_behave_test; + col4 +------ + 52 + 62 + 72 +(3 rows) + +SELECT col5 FROM colstore_behave_test; + col5 +------ + 42 + 82 + 92 +(3 rows) + +SELECT col6 FROM colstore_behave_test; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT col1, col3, col6 FROM colstore_behave_test; + col1 | col3 | col6 +------+------+------- + 1 | 12 | col6a + 2 | 22 | col6b + 3 | 32 | col6c +(3 rows) + +SELECT col6, col1, col3 FROM colstore_behave_test; + col6 | col1 | col3 +-------+------+------ + col6a | 1 | 12 + col6b | 2 | 22 + col6c | 3 | 32 +(3 rows) + +SELECT 1 FROM colstore_behave_test WHERE col1 >= 2; + ?column? +---------- + 1 + 1 +(2 rows) + +SELECT 2 FROM colstore_behave_test WHERE col1 IN (1,3); + ?column? +---------- + 2 + 2 +(2 rows) + +SELECT 3 FROM colstore_behave_test WHERE col2 = 'c'; + ?column? +---------- + 3 +(1 row) + +SELECT 4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + ?column? +---------- + 4 + 4 +(2 rows) + +SELECT 5 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + ?column? +---------- + 5 + 5 + 5 +(3 rows) + +SELECT 6 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + ?column? +---------- + 6 + 6 + 6 +(3 rows) + +SELECT 7 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + ?column? +---------- + 7 + 7 + 7 +(3 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 >= 2; + col1 +------ + 2 + 3 +(2 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 IN (1,3); + col1 +------ + 1 + 3 +(2 rows) + +SELECT col2 FROM colstore_behave_test WHERE col2 = 'c'; + col2 +------ + c +(1 row) + +SELECT col3, col4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col3 | col4 +------+------ + 12 | 52 + 22 | 62 +(2 rows) + +SELECT col6 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col1 >= 2; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col1 IN (1,3); + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col2 = 'c'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(1 row) + +SELECT * FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+--------------- + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/output/colstore_behave_test1.source b/src/test/regress/output/colstore_behave_test1.source new file mode 100644 index 0000000..d643bb0 --- /dev/null +++ b/src/test/regress/output/colstore_behave_test1.source @@ -0,0 +1,253 @@ +CREATE COLUMN STORE ACCESS METHOD test HANDLER vertical_cstore_handler; +CREATE TABLE colstore_behave_test +(col1 bigint +,col2 text +,col3 bigint COLUMN STORE col3 USING test +,col4 integer +,col5 smallint +,col6 text +,col7 char(10) not null +,col8 text +); +\i @abs_srcdir@/sql/colstore_behave.sql +-- Required behavior for all column stores +INSERT INTO colstore_behave_test +VALUES (1, 'a', 12, 52, 42, 'col6a', 'col7z', 'col8_1234567'); +INSERT INTO colstore_behave_test +VALUES (2, 'b', 22, 62, 82, 'col6b', 'col7y', 'col8_89101112'); +INSERT INTO colstore_behave_test +VALUES (3, 'c', 32, 72, 92, 'col6c', 'col7x', 'col8_13141516888'); +SELECT col1 FROM colstore_behave_test; + col1 +------ + 1 + 2 + 3 +(3 rows) + +SELECT col2 FROM colstore_behave_test; + col2 +------ + a + b + c +(3 rows) + +SELECT col3 FROM colstore_behave_test; + col3 +------ + 12 + 22 + 32 +(3 rows) + +SELECT col4 FROM colstore_behave_test; + col4 +------ + 52 + 62 + 72 +(3 rows) + +SELECT col5 FROM colstore_behave_test; + col5 +------ + 42 + 82 + 92 +(3 rows) + +SELECT col6 FROM colstore_behave_test; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT col1, col3, col6 FROM colstore_behave_test; + col1 | col3 | col6 +------+------+------- + 1 | 12 | col6a + 2 | 22 | col6b + 3 | 32 | col6c +(3 rows) + +SELECT col6, col1, col3 FROM colstore_behave_test; + col6 | col1 | col3 +-------+------+------ + col6a | 1 | 12 + col6b | 2 | 22 + col6c | 3 | 32 +(3 rows) + +SELECT 1 FROM colstore_behave_test WHERE col1 >= 2; + ?column? +---------- + 1 + 1 +(2 rows) + +SELECT 2 FROM colstore_behave_test WHERE col1 IN (1,3); + ?column? +---------- + 2 + 2 +(2 rows) + +SELECT 3 FROM colstore_behave_test WHERE col2 = 'c'; + ?column? +---------- + 3 +(1 row) + +SELECT 4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + ?column? +---------- + 4 + 4 +(2 rows) + +SELECT 5 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + ?column? +---------- + 5 + 5 + 5 +(3 rows) + +SELECT 6 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + ?column? +---------- + 6 + 6 + 6 +(3 rows) + +SELECT 7 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + ?column? +---------- + 7 + 7 + 7 +(3 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 >= 2; + col1 +------ + 2 + 3 +(2 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 IN (1,3); + col1 +------ + 1 + 3 +(2 rows) + +SELECT col2 FROM colstore_behave_test WHERE col2 = 'c'; + col2 +------ + c +(1 row) + +SELECT col3, col4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col3 | col4 +------+------ + 12 | 52 + 22 | 62 +(2 rows) + +SELECT col6 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col1 >= 2; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col1 IN (1,3); + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col2 = 'c'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(1 row) + +SELECT * FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+--------------- + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/output/colstore_behave_test2.source b/src/test/regress/output/colstore_behave_test2.source new file mode 100644 index 0000000..ff1c606 --- /dev/null +++ b/src/test/regress/output/colstore_behave_test2.source @@ -0,0 +1,252 @@ +CREATE TABLE colstore_behave_test +(col1 bigint +,col2 text +,col3 bigint +,col4 integer +,col5 smallint +,col6 text COLUMN STORE col6 USING test +,col7 char(10) not null COLUMN STORE col7 USING test +,col8 text COLUMN STORE col8 USING test +); +\i @abs_srcdir@/sql/colstore_behave.sql +-- Required behavior for all column stores +INSERT INTO colstore_behave_test +VALUES (1, 'a', 12, 52, 42, 'col6a', 'col7z', 'col8_1234567'); +INSERT INTO colstore_behave_test +VALUES (2, 'b', 22, 62, 82, 'col6b', 'col7y', 'col8_89101112'); +INSERT INTO colstore_behave_test +VALUES (3, 'c', 32, 72, 92, 'col6c', 'col7x', 'col8_13141516888'); +SELECT col1 FROM colstore_behave_test; + col1 +------ + 1 + 2 + 3 +(3 rows) + +SELECT col2 FROM colstore_behave_test; + col2 +------ + a + b + c +(3 rows) + +SELECT col3 FROM colstore_behave_test; + col3 +------ + 12 + 22 + 32 +(3 rows) + +SELECT col4 FROM colstore_behave_test; + col4 +------ + 52 + 62 + 72 +(3 rows) + +SELECT col5 FROM colstore_behave_test; + col5 +------ + 42 + 82 + 92 +(3 rows) + +SELECT col6 FROM colstore_behave_test; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT col1, col3, col6 FROM colstore_behave_test; + col1 | col3 | col6 +------+------+------- + 1 | 12 | col6a + 2 | 22 | col6b + 3 | 32 | col6c +(3 rows) + +SELECT col6, col1, col3 FROM colstore_behave_test; + col6 | col1 | col3 +-------+------+------ + col6a | 1 | 12 + col6b | 2 | 22 + col6c | 3 | 32 +(3 rows) + +SELECT 1 FROM colstore_behave_test WHERE col1 >= 2; + ?column? +---------- + 1 + 1 +(2 rows) + +SELECT 2 FROM colstore_behave_test WHERE col1 IN (1,3); + ?column? +---------- + 2 + 2 +(2 rows) + +SELECT 3 FROM colstore_behave_test WHERE col2 = 'c'; + ?column? +---------- + 3 +(1 row) + +SELECT 4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + ?column? +---------- + 4 + 4 +(2 rows) + +SELECT 5 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + ?column? +---------- + 5 + 5 + 5 +(3 rows) + +SELECT 6 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + ?column? +---------- + 6 + 6 + 6 +(3 rows) + +SELECT 7 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + ?column? +---------- + 7 + 7 + 7 +(3 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 >= 2; + col1 +------ + 2 + 3 +(2 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 IN (1,3); + col1 +------ + 1 + 3 +(2 rows) + +SELECT col2 FROM colstore_behave_test WHERE col2 = 'c'; + col2 +------ + c +(1 row) + +SELECT col3, col4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col3 | col4 +------+------ + 12 | 52 + 22 | 62 +(2 rows) + +SELECT col6 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col1 >= 2; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col1 IN (1,3); + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col2 = 'c'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(1 row) + +SELECT * FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+--------------- + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/output/colstore_behave_test3.source b/src/test/regress/output/colstore_behave_test3.source new file mode 100644 index 0000000..e0811eb --- /dev/null +++ b/src/test/regress/output/colstore_behave_test3.source @@ -0,0 +1,253 @@ +CREATE TABLE colstore_behave_test +(col1 bigint COLUMN STORE col1 USING test +,col2 text +,col3 bigint +,col4 integer +,col5 smallint +,col6 text +,col7 char(10) not null +,col8 text +,COLUMN STORE int USING test (col3, col4, col5) +); +\i @abs_srcdir@/sql/colstore_behave.sql +-- Required behavior for all column stores +INSERT INTO colstore_behave_test +VALUES (1, 'a', 12, 52, 42, 'col6a', 'col7z', 'col8_1234567'); +INSERT INTO colstore_behave_test +VALUES (2, 'b', 22, 62, 82, 'col6b', 'col7y', 'col8_89101112'); +INSERT INTO colstore_behave_test +VALUES (3, 'c', 32, 72, 92, 'col6c', 'col7x', 'col8_13141516888'); +SELECT col1 FROM colstore_behave_test; + col1 +------ + 1 + 2 + 3 +(3 rows) + +SELECT col2 FROM colstore_behave_test; + col2 +------ + a + b + c +(3 rows) + +SELECT col3 FROM colstore_behave_test; + col3 +------ + 12 + 22 + 32 +(3 rows) + +SELECT col4 FROM colstore_behave_test; + col4 +------ + 52 + 62 + 72 +(3 rows) + +SELECT col5 FROM colstore_behave_test; + col5 +------ + 42 + 82 + 92 +(3 rows) + +SELECT col6 FROM colstore_behave_test; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT col1, col3, col6 FROM colstore_behave_test; + col1 | col3 | col6 +------+------+------- + 1 | 12 | col6a + 2 | 22 | col6b + 3 | 32 | col6c +(3 rows) + +SELECT col6, col1, col3 FROM colstore_behave_test; + col6 | col1 | col3 +-------+------+------ + col6a | 1 | 12 + col6b | 2 | 22 + col6c | 3 | 32 +(3 rows) + +SELECT 1 FROM colstore_behave_test WHERE col1 >= 2; + ?column? +---------- + 1 + 1 +(2 rows) + +SELECT 2 FROM colstore_behave_test WHERE col1 IN (1,3); + ?column? +---------- + 2 + 2 +(2 rows) + +SELECT 3 FROM colstore_behave_test WHERE col2 = 'c'; + ?column? +---------- + 3 +(1 row) + +SELECT 4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + ?column? +---------- + 4 + 4 +(2 rows) + +SELECT 5 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + ?column? +---------- + 5 + 5 + 5 +(3 rows) + +SELECT 6 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + ?column? +---------- + 6 + 6 + 6 +(3 rows) + +SELECT 7 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + ?column? +---------- + 7 + 7 + 7 +(3 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 >= 2; + col1 +------ + 2 + 3 +(2 rows) + +SELECT col1 FROM colstore_behave_test WHERE col1 IN (1,3); + col1 +------ + 1 + 3 +(2 rows) + +SELECT col2 FROM colstore_behave_test WHERE col2 = 'c'; + col2 +------ + c +(1 row) + +SELECT col3, col4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col3 | col4 +------+------ + 12 | 52 + 22 | 62 +(2 rows) + +SELECT col6 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col6 +------- + col6a + col6b + col6c +(3 rows) + +SELECT col7 FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col7 +------------ + col7z + col7y + col7x +(3 rows) + +SELECT col8 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col8 +------------------ + col8_1234567 + col8_89101112 + col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col1 >= 2; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col1 IN (1,3); + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col2 = 'c'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(1 row) + +SELECT * FROM colstore_behave_test WHERE col3 =12 OR col4 =62; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+--------------- + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 +(2 rows) + +SELECT * FROM colstore_behave_test WHERE col6 LIKE 'col6%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col7 LIKE '%7%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +SELECT * FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + col1 | col2 | col3 | col4 | col5 | col6 | col7 | col8 +------+------+------+------+------+-------+------------+------------------ + 1 | a | 12 | 52 | 42 | col6a | col7z | col8_1234567 + 2 | b | 22 | 62 | 82 | col6b | col7y | col8_89101112 + 3 | c | 32 | 72 | 92 | col6c | col7x | col8_13141516888 +(3 rows) + +DROP TABLE colstore_behave_test; diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index b1bc7c7..582871f 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -62,6 +62,12 @@ test: create_index create_view # ---------- test: create_aggregate create_function_3 create_cast constraints triggers inherit create_table_like typed_table vacuum drop_if_exists updatable_views rolenames roleattributes +test: cstore +test: colstore_behave_control +test: colstore_behave_test1 +test: colstore_behave_test2 +test: colstore_behave_test3 + # ---------- # sanity_check does a vacuum, affecting the sort order of SELECT * # results. So it should not run parallel to other tests. diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index ade9ef1..f3d6989 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -62,6 +62,11 @@ test: create_misc test: create_operator test: create_index test: create_view +test: cstore +test: colstore_behave_control +test: colstore_behave_test1 +test: colstore_behave_test2 +test: colstore_behave_test3 test: create_aggregate test: create_function_3 test: create_cast @@ -75,6 +80,7 @@ test: drop_if_exists test: updatable_views test: rolenames test: roleattributes +test: cstore test: sanity_check test: errors test: select diff --git a/src/test/regress/sql/colstore_behave.sql b/src/test/regress/sql/colstore_behave.sql new file mode 100644 index 0000000..ce887c4 --- /dev/null +++ b/src/test/regress/sql/colstore_behave.sql @@ -0,0 +1,46 @@ +-- Required behavior for all column stores + +INSERT INTO colstore_behave_test +VALUES (1, 'a', 12, 52, 42, 'col6a', 'col7z', 'col8_1234567'); +INSERT INTO colstore_behave_test +VALUES (2, 'b', 22, 62, 82, 'col6b', 'col7y', 'col8_89101112'); +INSERT INTO colstore_behave_test +VALUES (3, 'c', 32, 72, 92, 'col6c', 'col7x', 'col8_13141516888'); + +SELECT col1 FROM colstore_behave_test; +SELECT col2 FROM colstore_behave_test; +SELECT col3 FROM colstore_behave_test; +SELECT col4 FROM colstore_behave_test; +SELECT col5 FROM colstore_behave_test; +SELECT col6 FROM colstore_behave_test; +SELECT col7 FROM colstore_behave_test; +SELECT col8 FROM colstore_behave_test; + +SELECT col1, col3, col6 FROM colstore_behave_test; +SELECT col6, col1, col3 FROM colstore_behave_test; + +SELECT 1 FROM colstore_behave_test WHERE col1 >= 2; +SELECT 2 FROM colstore_behave_test WHERE col1 IN (1,3); +SELECT 3 FROM colstore_behave_test WHERE col2 = 'c'; +SELECT 4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; +SELECT 5 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; +SELECT 6 FROM colstore_behave_test WHERE col7 LIKE '%7%'; +SELECT 7 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + +SELECT col1 FROM colstore_behave_test WHERE col1 >= 2; +SELECT col1 FROM colstore_behave_test WHERE col1 IN (1,3); +SELECT col2 FROM colstore_behave_test WHERE col2 = 'c'; +SELECT col3, col4 FROM colstore_behave_test WHERE col3 =12 OR col4 =62; +SELECT col6 FROM colstore_behave_test WHERE col6 LIKE 'col6%'; +SELECT col7 FROM colstore_behave_test WHERE col7 LIKE '%7%'; +SELECT col8 FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + +SELECT * FROM colstore_behave_test WHERE col1 >= 2; +SELECT * FROM colstore_behave_test WHERE col1 IN (1,3); +SELECT * FROM colstore_behave_test WHERE col2 = 'c'; +SELECT * FROM colstore_behave_test WHERE col3 =12 OR col4 =62; +SELECT * FROM colstore_behave_test WHERE col6 LIKE 'col6%'; +SELECT * FROM colstore_behave_test WHERE col7 LIKE '%7%'; +SELECT * FROM colstore_behave_test WHERE col8 LIKE 'col8%'; + + diff --git a/src/test/regress/sql/cstore.sql b/src/test/regress/sql/cstore.sql new file mode 100644 index 0000000..19a32e6 --- /dev/null +++ b/src/test/regress/sql/cstore.sql @@ -0,0 +1,262 @@ +CREATE COLUMN STORE ACCESS METHOD store_am_one HANDLER vertical_cstore_handler; +CREATE COLUMN STORE ACCESS METHOD store_am_two HANDLER vertical_cstore_handler; + +-- column-level column store definition + +-- missing USING clause +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE foo, + c INT +); + +-- missing column store name +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE USING store_am_one, + c INT +); + +-- missing USING and column store name +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE, + c INT +); + +-- missing column store name +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE USING store_am_one, + c INT +); + +-- unknown name of a column store AM +CREATE TABLE test_columnar_single_missing ( + a INT, + b INT COLUMN STORE foo USING no_store_am, + c INT +); + +-- conflicting column store name +CREATE TABLE test_columnar_single_conflict ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT COLUMN STORE foo USING store_am_one, + d INT +); + +-- correct definition (single store) +CREATE TABLE test_columnar_single_ok ( + a INT, + b INT COLUMN STORE foo1 USING store_am_one, + c INT +); + +\d test_columnar_single_ok + +-- correct definition (two stores) +CREATE TABLE test_columnar_single_ok2 ( + a INT, + b INT COLUMN STORE foo1 USING store_am_one, + c INT COLUMN STORE foo2 USING store_am_two, + d INT +); + +\d test_columnar_single_ok2 + +-- table-level column store definition + +-- no column list +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one +); + +-- empty column list +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one () +); + +-- invalid column in store +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one (z) +); + +-- unknown name of a column store AM +CREATE TABLE test_columnar_multi_missing ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING no_store_am (b,c) +); + +-- conflicting column store name +CREATE TABLE test_columnar_multi_conflict ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo USING store_am_one (a,b), + COLUMN STORE foo USING store_am_one (c,d) +); + +-- overlapping list of columns +CREATE TABLE test_columnar_multi_conflict2 ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo1 USING store_am_one (a,b), + COLUMN STORE foo2 USING store_am_two (b,c,d) +); + +-- correct definition (single store) +CREATE TABLE test_columnar_multi_ok ( + a INT, + b INT, + c INT, + COLUMN STORE foo USING store_am_one (a,b) +); + +\d test_columnar_multi_ok + +-- correct definition (two stores) +CREATE TABLE test_columnar_multi_ok2 ( + a INT, + b INT, + c INT, + d INT, + COLUMN STORE foo1 USING store_am_one (a,b), + COLUMN STORE foo2 USING store_am_one (c,d) +); + +\d test_columnar_multi_ok2 + +-- combination of column-level and table-level column stores + +-- conflicting column store name +CREATE TABLE test_columnar_multi_conflict ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT, + d INT, + COLUMN STORE foo USING store_am_one (c,d) +); + +-- overlapping list of columns +CREATE TABLE test_columnar_combi_conflict2 ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT, + d INT, + COLUMN STORE foo2 USING store_am_two (b,c,d) +); + +-- correct definition (two stores) +CREATE TABLE test_columnar_combi_ok ( + a INT, + b INT COLUMN STORE foo USING store_am_one, + c INT, + d INT, + COLUMN STORE foo2 USING store_am_one (c,d) +); + +\d test_columnar_combi_ok + +-- test cleanup +CREATE TABLE cstore_oids AS +SELECT cststoreid + FROM pg_cstore JOIN pg_class ON (pg_cstore.cstrelid = pg_class.oid) + WHERE relname IN ('test_columnar_single_ok', + 'test_columnar_single_ok2', + 'test_columnar_multi_ok', + 'test_columnar_multi_ok2', + 'test_columnar_combi_ok'); + +CREATE TABLE cstore_oids_2 AS +SELECT pg_class.oid + FROM pg_class JOIN cstore_oids ON (pg_class.oid = cstore_oids.cststoreid); + +DROP TABLE test_columnar_single_ok; +DROP TABLE test_columnar_single_ok2; +DROP TABLE test_columnar_multi_ok; +DROP TABLE test_columnar_multi_ok2; +DROP TABLE test_columnar_combi_ok; + +-- should return 0 +SELECT COUNT(*) FROM pg_class WHERE oid IN (SELECT cststoreid FROM cstore_oids); +SELECT COUNT(*) FROM pg_class WHERE oid IN (SELECT oid FROM cstore_oids_2); + +SELECT COUNT(*) FROM pg_cstore WHERE cststoreid IN (SELECT oid FROM cstore_oids); + +SELECT COUNT(*) FROM pg_attribute WHERE attrelid IN (SELECT cststoreid FROM cstore_oids); +SELECT COUNT(*) FROM pg_attribute WHERE attrelid IN (SELECT oid FROM cstore_oids_2); + +DROP TABLE cstore_oids; +DROP TABLE cstore_oids_2; + +-- INHERITANCE + +-- parent table with two column stores +CREATE TABLE parent_table ( + a INT, + b INT COLUMN STORE foo1 USING store_am_one, + c INT, + d INT, + e INT, + COLUMN STORE foo2 USING store_am_two (d,e) +); + +-- child table with two separate column stores +CREATE TABLE child_table_1 ( + f INT, + g INT COLUMN STORE foo1c USING store_am_one, + h INT, + i INT, + COLUMN STORE foo2c USING store_am_two(h,i) +) INHERITS (parent_table); + +\d child_table_1 + +-- child table with two column stores - one modifying, one redefining the parent +CREATE TABLE child_table_2 ( + f INT, + g INT COLUMN STORE foo1c USING store_am_one, -- new column store + h INT, + i INT, + COLUMN STORE foo2c USING store_am_two(b,h,i) -- redefines the parent colstore +) INHERITS (parent_table); + +\d child_table_2 + +-- child table with a single column store of the whole table +CREATE TABLE child_table_3 ( + f INT, + g INT, + h INT, + i INT, + COLUMN STORE foo1 USING store_am_one(a,b,c,d,e,f,g,h,i) +) INHERITS (parent_table); + +\d child_table_3 + +--- FIXME -- add tests with multiple inheritance + +DROP TABLE parent_table CASCADE; + +--- delete the fake cstore AM records +-- FIXME -- this should be a DROP command +DELETE FROM pg_cstore_am WHERE cstamname IN ('store_am_one', 'store_am_two'); diff --git a/src/test/regress/sql/sanity_check.sql b/src/test/regress/sql/sanity_check.sql index 0da838e..ca87cbc 100644 --- a/src/test/regress/sql/sanity_check.sql +++ b/src/test/regress/sql/sanity_check.sql @@ -10,7 +10,7 @@ VACUUM; -- temporarily disable fancy output, so catalog changes create less diff noise \a\t -SELECT relname, relhasindex +SELECT relname, relhasindex, relhascstore FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = relnamespace WHERE relkind = 'r' AND (nspname ~ '^pg_temp_') IS NOT TRUE ORDER BY relname;