From e51539968dec49235cf8579afb142a6fb3529967 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 18 Jun 2018 16:00:43 +0300 Subject: [PATCH 8/8] Add documentation for custom compression methods Signed-off-by: Ildus Kurbangaliev --- doc/src/sgml/catalogs.sgml | 19 ++- doc/src/sgml/compression-am.sgml | 178 +++++++++++++++++++++ doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/indexam.sgml | 2 +- doc/src/sgml/postgres.sgml | 1 + doc/src/sgml/ref/alter_table.sgml | 18 +++ doc/src/sgml/ref/create_access_method.sgml | 5 +- doc/src/sgml/ref/create_table.sgml | 13 ++ doc/src/sgml/storage.sgml | 6 +- 9 files changed, 236 insertions(+), 7 deletions(-) create mode 100644 doc/src/sgml/compression-am.sgml diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 0fd792ff1a..c8551c2fc3 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -57,7 +57,7 @@ pg_am - index access methods + access methods @@ -70,6 +70,11 @@ access method support functions + + pg_attr_compression + table columns compression relationships and options + + pg_attrdef column default values @@ -587,8 +592,10 @@ The catalog pg_am stores information about relation access methods. There is one row for each access method supported by the system. - Currently, only indexes have access methods. The requirements for index - access methods are discussed in detail in . + Currently, compression and index access methods are supported. + The requirements for index access methods are discussed in detail + in , for compression access methods + could be found in . @@ -892,6 +899,12 @@ + + <structname>pg_attr_compression</structname> + + pg_attr_compression + + <structname>pg_attrdef</structname> diff --git a/doc/src/sgml/compression-am.sgml b/doc/src/sgml/compression-am.sgml new file mode 100644 index 0000000000..e23d817910 --- /dev/null +++ b/doc/src/sgml/compression-am.sgml @@ -0,0 +1,178 @@ + + + + Compression Access Methods + + PostgreSQL supports two internal + built-in compression methods (pglz + and zlib), and also allows to add more custom compression + methods through compression access methods interface. + + + + Built-in Compression Access Methods + + These compression access methods are included in + PostgreSQL and don't need any external extensions. + +
+ Built-in Compression Access Methods + + + + Name + Options + + + + + pglz + + min_input_size (int), + max_input_size (int), + min_comp_rate (int), + first_success_by (int), + match_size_good (int), + match_size_drop (int) + + + + zlib + level (text), dict (text) + + + +
+ + Note that for zlib to work it should be installed in the + system and PostgreSQL should be compiled without + --without-zlib flag. + + + + + Basic API for compression methods + + + Each compression access method is described by a row in the + pg_am + system catalog. The pg_am entry + specifies a name and a handler function for the access + method. These entries can be created and deleted using the + and + SQL commands. + + + + A compression access method handler function must be declared to accept a + single argument of type internal and to return the + pseudo-type compression_am_handler. The argument is a dummy value that + simply serves to prevent handler functions from being called directly from + SQL commands. The result of the function must be a palloc'd struct of + type CompressionAmRoutine, which contains everything + that the core code needs to know to make use of the compression access method. + The CompressionAmRoutine struct, also called the access + method's API struct, contains pointers to support + functions for the access method. These support functions are plain C + functions and are not visible or callable at the SQL level. + The support functions are described in . + + + + The structure CompressionAmRoutine is defined thus: + +typedef struct CompressionAmRoutine +{ + NodeTag type; + + cmcheck_function cmcheck; /* can be NULL */ + cminitstate_function cminitstate; /* can be NULL */ + cmcompress_function cmcompress; + cmcompress_function cmdecompress; +} CompressionAmRoutine; + + + + + Compression Access Method Functions + + + The compression and auxiliary functions that an compression access + method must provide in CompressionAmRoutine are: + + + + +void +cmcheck (Form_pg_attribute att, List *options); + + Called when an attribute is linked with compression access method. Could + be used to check compatibility with the attribute and other additional + checks. + + + + Compression functions take special struct + CompressionAmOptions as first + parameter. This struct contains per backend cached state for each + attribute compression record. CompressionAmOptions is defined thus: + + +typedef struct CompressionAmOptions +{ + Oid acoid; /* Oid of attribute compression */ + Oid amoid; /* Oid of compression access method */ + List *acoptions; /* Parsed options, used for comparison */ + CompressionAmRoutine *amroutine; /* compression access method routine */ + + /* result of cminitstate function will be put here */ + void *acstate; +} CompressionAmOptions; + + + + + The acstate field is used to keep temporary state + between compression functions calls and stores the result of + cminitstate function. It could be useful to store + the parsed view of the compression options. + + + + Note that any invalidation of pg_attr_compression relation + will cause all the cached acstate options cleared. + They will be recreated on the next compression functions calls. + + + + +void * +cminitstate (Oid acoid, List *options); + + Called when CompressionAmOptions is being + initialized. Can return a pointer to memory that will be passed between + compression functions calls. + + + + +struct varlena * +cmcompress (CompressionAmOptions *cmoptions, + const struct varlena *value); + + Function is used to compress varlena. Could return NULL if data is + incompressible. If it returns varlena bigger than original the core will + not use it. + + + + +struct varlena * +cmdecompress (CompressionAmOptions *cmoptions, + const struct varlena *value); + + Function is used to decompress varlena. + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index a03ea1427b..8f482a60dd 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -90,6 +90,7 @@ + diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index 05102724ea..54110050d1 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -47,7 +47,7 @@ Basic API Structure for Indexes - Each index access method is described by a row in the + Each index access method is described by a row with INDEX type in the pg_am system catalog. The pg_am entry specifies a name and a handler function for the access diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 96d196d229..17f2cc98b3 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -251,6 +251,7 @@ &custom-scan; &geqo; &indexam; + &compression-am; &generic-wal; &btree; &gist; diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index e360728c02..09ad7c8aa2 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -53,6 +53,7 @@ ALTER TABLE [ IF EXISTS ] name ALTER [ COLUMN ] column_name SET ( attribute_option = value [, ... ] ) ALTER [ COLUMN ] column_name RESET ( attribute_option [, ... ] ) ALTER [ COLUMN ] column_name SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + ALTER [ COLUMN ] column_name SET COMPRESSION compression_am [ WITH (compression_am_options) ] [ PRESERVE (compression_preserve_list) ] ADD table_constraint [ NOT VALID ] ADD table_constraint_using_index ALTER CONSTRAINT constraint_name [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -359,6 +360,23 @@ WITH ( MODULUS numeric_literal, REM + + + SET COMPRESSION compression_method_name [ WITH (compression_method_options) ] [ PRESERVE (compression_preserve_list) ] + + + + This form adds compression to a column. Compression access method should be + created with . If compression + method has options they could be specified with WITH + parameter. The PRESERVE list contains list of compression access methods + used on the column and determines which of them should be kept on the + column. Without PRESERVE or partial list of compression methods table + will be rewritten. + + + + ADD table_constraint [ NOT VALID ] diff --git a/doc/src/sgml/ref/create_access_method.sgml b/doc/src/sgml/ref/create_access_method.sgml index 851c5e63be..a35005aca3 100644 --- a/doc/src/sgml/ref/create_access_method.sgml +++ b/doc/src/sgml/ref/create_access_method.sgml @@ -61,7 +61,7 @@ CREATE ACCESS METHOD name This clause specifies the type of access method to define. - Only INDEX is supported at present. + INDEX and COMPRESSION types are supported at present. @@ -79,6 +79,9 @@ CREATE ACCESS METHOD name be index_am_handler. The C-level API that the handler function must implement varies depending on the type of access method. The index access method API is described in . + For COMPRESSION access methods, the type must be + compression_am_handler. The compression access method API + is described in . diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index e94fe2c3b6..2d192148de 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -65,6 +65,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] | UNIQUE index_parameters | PRIMARY KEY index_parameters | + COMPRESSION compression_access_method [ WITH (compression_am_options) ] | REFERENCES reftable [ ( refcolumn ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE referential_action ] [ ON UPDATE referential_action ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -920,6 +921,18 @@ WITH ( MODULUS numeric_literal, REM + + COMPRESSION compression_access_method [ WITH (compression_am_options) ] + + + This clause adds compression to a column. Compression method could be + created with . If compression + method has options they could be specified by WITH + parameter. + + + + EXCLUDE [ USING index_method ] ( exclude_element WITH operator [, ... ] ) index_parameters [ WHERE ( predicate ) ] diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index cbdad0c3fb..89a5889d0f 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -385,10 +385,12 @@ Further details appear in . -The compression technique used for either in-line or out-of-line compressed +The default compression technique used for either in-line or out-of-line compressed data is a fairly simple and very fast member of the LZ family of compression techniques. See -src/common/pg_lzcompress.c for the details. +src/common/pg_lzcompress.c for the details. Also custom +compressions could be used. Look at for +more information. -- 2.21.0