From 2e5cbf2dadbdd4c7bc389e9fdc03f80705d21673 Mon Sep 17 00:00:00 2001 From: amit Date: Thu, 28 Jul 2016 13:40:02 +0900 Subject: [PATCH 8/8] Update DDL Partitioning chapter to reflect new developments. --- doc/src/sgml/ddl.sgml | 402 ++++++++++--------------------------------------- 1 files changed, 83 insertions(+), 319 deletions(-) diff --git a/doc/src/sgml/ddl.sgml b/doc/src/sgml/ddl.sgml index 157512c..288989b 100644 --- a/doc/src/sgml/ddl.sgml +++ b/doc/src/sgml/ddl.sgml @@ -2771,7 +2771,7 @@ VALUES ('Albany', NULL, NULL, 'NY'); Bulk loads and deletes can be accomplished by adding or removing partitions, if that requirement is planned into the partitioning design. - ALTER TABLE NO INHERIT and DROP TABLE are + ALTER TABLE DETACH PARTITION and DROP TABLE are both far faster than a bulk operation. These commands also entirely avoid the VACUUM overhead caused by a bulk DELETE. @@ -2793,12 +2793,15 @@ VALUES ('Albany', NULL, NULL, 'NY'); - Currently, PostgreSQL supports partitioning - via table inheritance. Each partition must be created as a child - table of a single parent table. The parent table itself is normally - empty; it exists just to represent the entire data set. You should be - familiar with inheritance (see ) before - attempting to set up partitioning. + Currently, PostgreSQL provides a way to + specify the partition key of table along with two methods of partitioning + to choose from. Individual partitions of a partitioned table are created + using separate CREATE TABLE commands where you must specify + the partition bound such that it does not overlap with any existing + partitions of the parent table. The parent table itself is empty; + it exists just to represent the entire data set. See and + for more details on the exact syntax to use for above mentioned commands. @@ -2842,59 +2845,22 @@ VALUES ('Albany', NULL, NULL, 'NY'); - Create the master table, from which all of the - partitions will inherit. + Create the partitioned table. This table will contain no data. Do not define any check constraints on this table, unless you intend them to be applied equally to all partitions. There is no point - in defining any indexes or unique constraints on it, either. + in defining any indexes or unique constraints on it, either, + since the notion of global uniqueness is not yet implemented. - Create several child tables that each inherit from - the master table. Normally, these tables will not add any columns - to the set inherited from the master. - - - - We will refer to the child tables as partitions, though they - are in every way normal PostgreSQL tables - (or, possibly, foreign tables). - - - - - - Add table constraints to the partition tables to define the - allowed key values in each partition. - - - - Typical examples would be: - -CHECK ( x = 1 ) -CHECK ( county IN ( 'Oxfordshire', 'Buckinghamshire', 'Warwickshire' )) -CHECK ( outletID >= 100 AND outletID < 200 ) - - Ensure that the constraints guarantee that there is no overlap - between the key values permitted in different partitions. A common - mistake is to set up range constraints like: - -CHECK ( outletID BETWEEN 100 AND 200 ) -CHECK ( outletID BETWEEN 200 AND 300 ) - - This is wrong since it is not clear which partition the key value - 200 belongs in. - - - - Note that there is no difference in - syntax between range and list partitioning; those terms are - descriptive only. + Create several partitions of the above created + partitioned table. Partitions are in every way normal + PostgreSQL tables (or, possibly, foreign tables). @@ -2911,8 +2877,10 @@ CHECK ( outletID BETWEEN 200 AND 300 ) - Optionally, define a trigger or rule to redirect data inserted into - the master table to the appropriate partition. + Note that a data row inserted into the master table will be mapped + to and stored in the appropriate partition. If some row does not + fall within any of existing partitions, an error will be thrown. + You must create the missing partition explicitly. @@ -2940,7 +2908,7 @@ CREATE TABLE measurement ( logdate date not null, peaktemp int, unitsales int -); +) PARTITION BY RANGE (logdate); We know that most queries will access just the last week's, month's or @@ -2971,12 +2939,12 @@ CREATE TABLE measurement ( Next we create one partition for each active month: -CREATE TABLE measurement_y2006m02 ( ) INHERITS (measurement); -CREATE TABLE measurement_y2006m03 ( ) INHERITS (measurement); +CREATE TABLE measurement_y2016m07 PARTITION OF measurement FOR VALUES FROM ('2016-07-01') TO ('2016-08-01'); +CREATE TABLE measurement_y2016m08 PARTITION OF measurement FOR VALUES FROM ('2016-08-01') TO ('2016-09-01'); ... -CREATE TABLE measurement_y2007m11 ( ) INHERITS (measurement); -CREATE TABLE measurement_y2007m12 ( ) INHERITS (measurement); -CREATE TABLE measurement_y2008m01 ( ) INHERITS (measurement); +CREATE TABLE measurement_y2017m04 PARTITION OF measurement FOR VALUES FROM ('2017-04-01') TO ('2017-05-01'); +CREATE TABLE measurement_y2017m05 PARTITION OF measurement FOR VALUES FROM ('2017-05-01') TO ('2017-06-01'); +CREATE TABLE measurement_y2017m06 PARTITION OF measurement FOR VALUES FROM ('2017-06-01') TO ('2017-07-01'); Each of the partitions are complete tables in their own right, @@ -2986,36 +2954,9 @@ CREATE TABLE measurement_y2008m01 ( ) INHERITS (measurement); This solves one of our problems: deleting old data. Each - month, all we will need to do is perform a DROP - TABLE on the oldest child table and create a new - child table for the new month's data. - - - - - - We must provide non-overlapping table constraints. Rather than - just creating the partition tables as above, the table creation - script should really be: - - -CREATE TABLE measurement_y2006m02 ( - CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -) INHERITS (measurement); -CREATE TABLE measurement_y2006m03 ( - CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) -) INHERITS (measurement); -... -CREATE TABLE measurement_y2007m11 ( - CHECK ( logdate >= DATE '2007-11-01' AND logdate < DATE '2007-12-01' ) -) INHERITS (measurement); -CREATE TABLE measurement_y2007m12 ( - CHECK ( logdate >= DATE '2007-12-01' AND logdate < DATE '2008-01-01' ) -) INHERITS (measurement); -CREATE TABLE measurement_y2008m01 ( - CHECK ( logdate >= DATE '2008-01-01' AND logdate < DATE '2008-02-01' ) -) INHERITS (measurement); - + month, all we will need to do is perform a ALTER TABLE + measurement DETACH PARTITION on the oldest child table + and create a new partition for the new month's data. @@ -3024,110 +2965,19 @@ CREATE TABLE measurement_y2008m01 ( We probably need indexes on the key columns too: -CREATE INDEX measurement_y2006m02_logdate ON measurement_y2006m02 (logdate); -CREATE INDEX measurement_y2006m03_logdate ON measurement_y2006m03 (logdate); +CREATE INDEX measurement_y2016m07_logdate ON measurement_y2016m07 (logdate); +CREATE INDEX measurement_y2016m08_logdate ON measurement_y2016m08 (logdate); ... -CREATE INDEX measurement_y2007m11_logdate ON measurement_y2007m11 (logdate); -CREATE INDEX measurement_y2007m12_logdate ON measurement_y2007m12 (logdate); -CREATE INDEX measurement_y2008m01_logdate ON measurement_y2008m01 (logdate); +CREATE INDEX measurement_y2017m04_logdate ON measurement_y2017m04 (logdate); +CREATE INDEX measurement_y2017m05_logdate ON measurement_y2017m05 (logdate); +CREATE INDEX measurement_y2017m06_logdate ON measurement_y2017m06 (logdate); We choose not to add further indexes at this time. - - - - We want our application to be able to say INSERT INTO - measurement ... and have the data be redirected into the - appropriate partition table. We can arrange that by attaching - a suitable trigger function to the master table. - If data will be added only to the latest partition, we can - use a very simple trigger function: - - -CREATE OR REPLACE FUNCTION measurement_insert_trigger() -RETURNS TRIGGER AS $$ -BEGIN - INSERT INTO measurement_y2008m01 VALUES (NEW.*); - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - - - After creating the function, we create a trigger which - calls the trigger function: - - -CREATE TRIGGER insert_measurement_trigger - BEFORE INSERT ON measurement - FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); - - - We must redefine the trigger function each month so that it always - points to the current partition. The trigger definition does - not need to be updated, however. - - - - We might want to insert data and have the server automatically - locate the partition into which the row should be added. We - could do this with a more complex trigger function, for example: - - -CREATE OR REPLACE FUNCTION measurement_insert_trigger() -RETURNS TRIGGER AS $$ -BEGIN - IF ( NEW.logdate >= DATE '2006-02-01' AND - NEW.logdate < DATE '2006-03-01' ) THEN - INSERT INTO measurement_y2006m02 VALUES (NEW.*); - ELSIF ( NEW.logdate >= DATE '2006-03-01' AND - NEW.logdate < DATE '2006-04-01' ) THEN - INSERT INTO measurement_y2006m03 VALUES (NEW.*); - ... - ELSIF ( NEW.logdate >= DATE '2008-01-01' AND - NEW.logdate < DATE '2008-02-01' ) THEN - INSERT INTO measurement_y2008m01 VALUES (NEW.*); - ELSE - RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; - END IF; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - - - The trigger definition is the same as before. - Note that each IF test must exactly match the - CHECK constraint for its partition. - - - - While this function is more complex than the single-month case, - it doesn't need to be updated as often, since branches can be - added in advance of being needed. - - - - - In practice it might be best to check the newest partition first, - if most inserts go into that partition. For simplicity we have - shown the trigger's tests in the same order as in other parts - of this example. - - - - - - As we can see, a complex partitioning scheme could require a - substantial amount of DDL. In the above example we would be - creating a new partition each month, so it might be wise to write a - script that generates the required DDL automatically. - - @@ -3145,22 +2995,17 @@ LANGUAGE plpgsql; - The simplest option for removing old data is simply to drop the partition + The simplest option for removing old data is simply detach the partition that is no longer necessary: -DROP TABLE measurement_y2006m02; +ALTER TABLE measurement DETACH PARTITION measurement_y2016m07; + This can very quickly delete millions of records because it doesn't have to individually delete every record. - - - Another option that is often preferable is to remove the partition from - the partitioned table but retain access to it as a table in its own - right: - -ALTER TABLE measurement_y2006m02 NO INHERIT measurement; - + The detached partition continues to exist as a regular table, which if + necessary can be dropped using regular DROP TABLE command. This allows further operations to be performed on the data before it is dropped. For example, this is often a useful time to back up the data using COPY, pg_dump, or @@ -3175,9 +3020,7 @@ ALTER TABLE measurement_y2006m02 NO INHERIT measurement; were created above: -CREATE TABLE measurement_y2008m02 ( - CHECK ( logdate >= DATE '2008-02-01' AND logdate < DATE '2008-03-01' ) -) INHERITS (measurement); +CREATE TABLE measurement_y2017m07 PARTITION OF measurement FOR VALUES FROM ('2017-07-01') TO ('2017-08-01'); As an alternative, it is sometimes more convenient to create the @@ -3186,13 +3029,15 @@ CREATE TABLE measurement_y2008m02 ( transformed prior to it appearing in the partitioned table: -CREATE TABLE measurement_y2008m02 +CREATE TABLE measurement_y2017m07 (LIKE measurement INCLUDING DEFAULTS INCLUDING CONSTRAINTS); -ALTER TABLE measurement_y2008m02 ADD CONSTRAINT y2008m02 - CHECK ( logdate >= DATE '2008-02-01' AND logdate < DATE '2008-03-01' ); -\copy measurement_y2008m02 from 'measurement_y2008m02' +ALTER TABLE measurement_y2017m07 ADD CONSTRAINT y2017m07 + CHECK ( logdate >= DATE '2017-07-01' AND logdate < DATE '2017-08-01' ); +\copy measurement_y2017m07 from 'measurement_y2017m07' +ALTER TABLE measurement_y2017m07 DROP CONSTRAINT y2017m07; -- possibly some other data preparation work -ALTER TABLE measurement_y2008m02 INHERIT measurement; +ALTER TABLE measurement + ATTACH PARTITION measurement_y2017m07 FOR VALUES FROM ('2017-07-01') TO ('2017-08-01'); @@ -3211,7 +3056,7 @@ ALTER TABLE measurement_y2008m02 INHERIT measurement; SET constraint_exclusion = on; -SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; +SELECT count(*) FROM measurement WHERE logdate >= DATE '2017-01-01'; Without constraint exclusion, the above query would scan each of @@ -3220,7 +3065,9 @@ SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; partition and try to prove that the partition need not be scanned because it could not contain any rows meeting the query's WHERE clause. When the planner can prove this, it - excludes the partition from the query plan. + excludes the partition from the query plan. Note that the aforementioned + constraints need not be explicitly created; they are internally derived + from the partition bound metadata. @@ -3230,23 +3077,23 @@ SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; SET constraint_exclusion = off; -EXPLAIN SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; - - QUERY PLAN ------------------------------------------------------------------------------------------------ - Aggregate (cost=158.66..158.68 rows=1 width=0) - -> Append (cost=0.00..151.88 rows=2715 width=0) - -> Seq Scan on measurement (cost=0.00..30.38 rows=543 width=0) - Filter: (logdate >= '2008-01-01'::date) - -> Seq Scan on measurement_y2006m02 measurement (cost=0.00..30.38 rows=543 width=0) - Filter: (logdate >= '2008-01-01'::date) - -> Seq Scan on measurement_y2006m03 measurement (cost=0.00..30.38 rows=543 width=0) - Filter: (logdate >= '2008-01-01'::date) +EXPLAIN SELECT count(*) FROM measurement WHERE logdate >= DATE '2018-07-01'; + + QUERY PLAN +----------------------------------------------------------------------------------- + Aggregate (cost=866.69..866.70 rows=1 width=8) + -> Append (cost=0.00..828.12 rows=15426 width=0) + -> Seq Scan on measurement (cost=0.00..0.00 rows=1 width=0) + Filter: (logdate >= '2017-01-01'::date) + -> Seq Scan on measurement_y2016m07 (cost=0.00..33.12 rows=617 width=0) + Filter: (logdate >= '2017-01-01'::date) + -> Seq Scan on measurement_y2016m08 (cost=0.00..33.12 rows=617 width=0) + Filter: (logdate >= '2017-01-01'::date) ... - -> Seq Scan on measurement_y2007m12 measurement (cost=0.00..30.38 rows=543 width=0) - Filter: (logdate >= '2008-01-01'::date) - -> Seq Scan on measurement_y2008m01 measurement (cost=0.00..30.38 rows=543 width=0) - Filter: (logdate >= '2008-01-01'::date) + -> Seq Scan on measurement_y2018m06 (cost=0.00..33.12 rows=617 width=0) + Filter: (logdate >= '2017-01-01'::date) + -> Seq Scan on measurement_y2018m07 (cost=0.00..33.12 rows=617 width=0) + Filter: (logdate >= '2017-01-01'::date) Some or all of the partitions might use index scans instead of @@ -3257,15 +3104,15 @@ EXPLAIN SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; SET constraint_exclusion = on; -EXPLAIN SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; - QUERY PLAN ------------------------------------------------------------------------------------------------ - Aggregate (cost=63.47..63.48 rows=1 width=0) - -> Append (cost=0.00..60.75 rows=1086 width=0) - -> Seq Scan on measurement (cost=0.00..30.38 rows=543 width=0) - Filter: (logdate >= '2008-01-01'::date) - -> Seq Scan on measurement_y2008m01 measurement (cost=0.00..30.38 rows=543 width=0) - Filter: (logdate >= '2008-01-01'::date) +EXPLAIN SELECT count(*) FROM measurement WHERE logdate >= DATE '2018-07-01'; + QUERY PLAN +----------------------------------------------------------------------------------- + Aggregate (cost=34.67..34.68 rows=1 width=8) + -> Append (cost=0.00..33.12 rows=618 width=0) + -> Seq Scan on measurement (cost=0.00..0.00 rows=1 width=0) + Filter: (logdate >= '2018-07-01'::date) + -> Seq Scan on measurement_y2018m07 (cost=0.00..33.12 rows=617 width=0) + Filter: (logdate >= '2018-07-01'::date) @@ -3292,93 +3139,22 @@ EXPLAIN SELECT count(*) FROM measurement WHERE logdate >= DATE '2008-01-01'; - - Alternative Partitioning Methods - - - A different approach to redirecting inserts into the appropriate - partition table is to set up rules, instead of a trigger, on the - master table. For example: - - -CREATE RULE measurement_insert_y2006m02 AS -ON INSERT TO measurement WHERE - ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) -DO INSTEAD - INSERT INTO measurement_y2006m02 VALUES (NEW.*); -... -CREATE RULE measurement_insert_y2008m01 AS -ON INSERT TO measurement WHERE - ( logdate >= DATE '2008-01-01' AND logdate < DATE '2008-02-01' ) -DO INSTEAD - INSERT INTO measurement_y2008m01 VALUES (NEW.*); - - - A rule has significantly more overhead than a trigger, but the overhead - is paid once per query rather than once per row, so this method might be - advantageous for bulk-insert situations. In most cases, however, the - trigger method will offer better performance. - - - - Be aware that COPY ignores rules. If you want to - use COPY to insert data, you'll need to copy into the correct - partition table rather than into the master. COPY does fire - triggers, so you can use it normally if you use the trigger approach. - - - - Another disadvantage of the rule approach is that there is no simple - way to force an error if the set of rules doesn't cover the insertion - date; the data will silently go into the master table instead. - - - - Partitioning can also be arranged using a UNION ALL - view, instead of table inheritance. For example, - - -CREATE VIEW measurement AS - SELECT * FROM measurement_y2006m02 -UNION ALL SELECT * FROM measurement_y2006m03 -... -UNION ALL SELECT * FROM measurement_y2007m11 -UNION ALL SELECT * FROM measurement_y2007m12 -UNION ALL SELECT * FROM measurement_y2008m01; - - - However, the need to recreate the view adds an extra step to adding and - dropping individual partitions of the data set. In practice this - method has little to recommend it compared to using inheritance. - - - - Caveats The following caveats apply to partitioned tables: - - - There is no automatic way to verify that all of the - CHECK constraints are mutually - exclusive. It is safer to create code that generates - partitions and creates and/or modifies associated objects than - to write each by hand. - - The schemes shown here assume that the partition key column(s) of a row never change, or at least do not change enough to require it to move to another partition. An UPDATE that attempts - to do that will fail because of the CHECK constraints. - If you need to handle such cases, you can put suitable update triggers - on the partition tables, but it makes management of the structure - much more complicated. + to do that will fail because of applying internally created CHECK + constraints. If you need to handle such cases, you can put suitable + update triggers on the partition tables, but it makes management of the + structure much more complicated. @@ -3397,9 +3173,9 @@ ANALYZE measurement; INSERT statements with ON CONFLICT - clauses are unlikely to work as expected, as the ON CONFLICT - action is only taken in case of unique violations on the specified - target relation, not its child relations. + clauses are currently unsupported on partitioned tables as there is + currently no reliable way to check global uniqueness across all the + partitions. @@ -3423,18 +3199,6 @@ ANALYZE measurement; - Keep the partitioning constraints simple, else the planner may not be - able to prove that partitions don't need to be visited. Use simple - equality conditions for list partitioning, or simple - range tests for range partitioning, as illustrated in the preceding - examples. A good rule of thumb is that partitioning constraints should - contain only comparisons of the partitioning column(s) to constants - using B-tree-indexable operators. - - - - - All constraints on all partitions of the master table are examined during constraint exclusion, so large numbers of partitions are likely to increase query planning time considerably. Partitioning using -- 1.7.1