diff --git a/src/backend/commands/tsearchcmds.c b/src/backend/commands/tsearchcmds.c index 3753e32b2c..ef6cabcc1e 100644 --- a/src/backend/commands/tsearchcmds.c +++ b/src/backend/commands/tsearchcmds.c @@ -39,6 +39,7 @@ #include "nodes/makefuncs.h" #include "parser/parse_func.h" #include "tsearch/ts_cache.h" +#include "tsearch/ts_shared.h" #include "tsearch/ts_utils.h" #include "utils/builtins.h" #include "utils/fmgroids.h" @@ -510,6 +511,7 @@ RemoveTSDictionaryById(Oid dictId) { Relation relation; HeapTuple tup; + DictPointerData dict; relation = heap_open(TSDictionaryRelationId, RowExclusiveLock); @@ -521,6 +523,18 @@ RemoveTSDictionaryById(Oid dictId) CatalogTupleDelete(relation, &tup->t_self); + /* + * We need to release the dictionary's DSM segment. The segment still may + * leak. It may happen if some backend used the dictionary before dropping, + * the backend will hold its DSM segment till disconnecting or calling + * lookup_ts_dictionary_cache(). + */ + dict.id = dictId; + dict.xmin = HeapTupleHeaderGetRawXmin(tup->t_data); + dict.xmax = HeapTupleHeaderGetRawXmax(tup->t_data); + dict.tid = tup->t_self; + ts_dict_shmem_release(&dict, true); + ReleaseSysCache(tup); heap_close(relation, RowExclusiveLock); @@ -544,6 +558,7 @@ AlterTSDictionary(AlterTSDictionaryStmt *stmt) bool repl_null[Natts_pg_ts_dict]; bool repl_repl[Natts_pg_ts_dict]; ObjectAddress address; + DictPointerData dict; dictId = get_ts_dict_oid(stmt->dictname, false); @@ -630,6 +645,18 @@ AlterTSDictionary(AlterTSDictionaryStmt *stmt) ObjectAddressSet(address, TSDictionaryRelationId, dictId); + /* + * We need to release the dictionary's DSM segment. The segment isn't valid + * anymor. The segment still may leak. It may happen if some backend used + * the dictionary before dropping, the backend will hold its DSM segment + * till disconnecting or calling lookup_ts_dictionary_cache(). + */ + dict.id = dictId; + dict.xmin = HeapTupleHeaderGetRawXmin(tup->t_data); + dict.xmax = HeapTupleHeaderGetRawXmax(tup->t_data); + dict.tid = tup->t_self; + ts_dict_shmem_release(&dict, true); + /* * NOTE: because we only support altering the options, not the template, * there is no need to update dependencies. This might have to change if diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 0c86a581c0..c7dce8cac5 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -44,6 +44,7 @@ #include "storage/procsignal.h" #include "storage/sinvaladt.h" #include "storage/spin.h" +#include "tsearch/ts_shared.h" #include "utils/backend_random.h" #include "utils/snapmgr.h" @@ -150,6 +151,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port) size = add_size(size, SyncScanShmemSize()); size = add_size(size, AsyncShmemSize()); size = add_size(size, BackendRandomShmemSize()); + size = add_size(size, TsearchShmemSize()); #ifdef EXEC_BACKEND size = add_size(size, ShmemBackendArraySize()); #endif @@ -271,6 +273,11 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port) AsyncShmemInit(); BackendRandomShmemInit(); + /* + * Set up shared memory to tsearch + */ + TsearchShmemInit(); + #ifdef EXEC_BACKEND /* diff --git a/src/backend/tsearch/Makefile b/src/backend/tsearch/Makefile index 227468ae9e..860cd196e9 100644 --- a/src/backend/tsearch/Makefile +++ b/src/backend/tsearch/Makefile @@ -26,7 +26,7 @@ DICTFILES_PATH=$(addprefix dicts/,$(DICTFILES)) OBJS = ts_locale.o ts_parse.o wparser.o wparser_def.o dict.o \ dict_simple.o dict_synonym.o dict_thesaurus.o \ dict_ispell.o regis.o spell.o \ - to_tsany.o ts_selfuncs.o ts_typanalyze.o ts_utils.o + to_tsany.o ts_selfuncs.o ts_shared.o ts_typanalyze.o ts_utils.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/tsearch/ts_shared.c b/src/backend/tsearch/ts_shared.c new file mode 100644 index 0000000000..590e93df8e --- /dev/null +++ b/src/backend/tsearch/ts_shared.c @@ -0,0 +1,384 @@ +/*------------------------------------------------------------------------- + * + * ts_shared.c + * tsearch shared memory management + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/ts_shared.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/hash.h" +#include "lib/dshash.h" +#include "miscadmin.h" +#include "storage/lwlock.h" +#include "storage/shmem.h" +#include "tsearch/ts_shared.h" +#include "utils/hashutils.h" +#include "utils/memutils.h" + + +/* + * Hash table entries key. + */ +typedef struct +{ + Oid db_id; + DictPointerData dict; +} TsearchDictKey; + +/* + * Hash table entries representing shared dictionaries. + */ +typedef struct +{ + TsearchDictKey key; + dsm_handle dict_dsm; + + /* How many backends have DSM mapping */ + uint32 refcnt; +} TsearchDictEntry; + +static dshash_table *dict_table = NULL; + +/* + * Information about the main shmem segment, used to coordinate + * access to the hash table and dictionaries. + */ +typedef struct +{ + dsa_handle area; + dshash_table_handle dict_table_handle; + + LWLock lock; +} TsearchCtlData; + +static TsearchCtlData *tsearch_ctl; + +static int tsearch_dict_cmp(const void *a, const void *b, size_t size, + void *arg); +static uint32 tsearch_dict_hash(const void *a, size_t size, void *arg); +static void init_dict_table(void); + +/* Parameters for dict_table */ +static const dshash_parameters dict_table_params ={ + sizeof(TsearchDictKey), + sizeof(TsearchDictEntry), + tsearch_dict_cmp, + tsearch_dict_hash, + LWTRANCHE_TSEARCH_TABLE +}; + +/* + * Build the dictionary using allocate_cb callback. + * + * Firstly try to find the dictionary in shared hash table. If it was built by + * someone earlier just return its location in DSM. + * + * init_data: an argument used within a template's init method. + * allocate_cb: function to build the dictionary, if it wasn't found in DSM. + * + * Returns address in the dynamic shared memory segment or in backend memory. + */ +void * +ts_dict_shmem_location(DictInitData *init_data, + ts_dict_build_callback allocate_cb) +{ + TsearchDictKey key; + TsearchDictEntry *entry; + bool found; + dsm_segment *seg; + void *dict, + *dict_location; + Size dict_size; + + init_dict_table(); + + /* + * Build the dictionary in backend's memory if dictid is invalid (it may + * happen if the dicionary's init method was called within + * verify_dictoptions()). + */ + if (!OidIsValid(init_data->dict.id)) + { + dict = allocate_cb(init_data->dict_options, &dict_size); + + return dict; + } + + /* Set up key for hashtable search */ + key.db_id = MyDatabaseId; + key.dict = init_data->dict; + + /* Try to find an entry in the hash table */ + entry = (TsearchDictEntry *) dshash_find(dict_table, &key, false); + + if (entry) + { + seg = dsm_find_mapping(entry->dict_dsm); + if (!seg) + { + seg = dsm_attach(entry->dict_dsm); + /* Remain attached until end of session */ + dsm_pin_mapping(seg); + } + + entry->refcnt++; + dshash_release_lock(dict_table, entry); + + return dsm_segment_address(seg); + } + + /* Dictionary haven't been loaded into memory yet */ + entry = (TsearchDictEntry *) dshash_find_or_insert(dict_table, &key, + &found); + + if (found) + { + /* + * Someone concurrently inserted a dictionary entry since the first time + * we checked. + */ + seg = dsm_attach(entry->dict_dsm); + + /* Remain attached until end of session */ + dsm_pin_mapping(seg); + + entry->refcnt++; + dshash_release_lock(dict_table, entry); + + return dsm_segment_address(seg); + } + + /* Build the dictionary */ + dict = allocate_cb(init_data->dict_options, &dict_size); + + /* At least, allocate a DSM segment for the compiled dictionary */ + seg = dsm_create(dict_size, 0); + dict_location = dsm_segment_address(seg); + memcpy(dict_location, dict, dict_size); + + pfree(dict); + + entry->key = key; + entry->dict_dsm = dsm_segment_handle(seg); + entry->refcnt = 1; + + /* Remain attached until end of postmaster */ + dsm_pin_segment(seg); + /* Remain attached until end of session */ + dsm_pin_mapping(seg); + + dshash_release_lock(dict_table, entry); + + return dsm_segment_address(seg); +} + +/* + * Release memory occupied by the dictionary. Function just unpins DSM mapping. + * If nobody else hasn't mapping to this DSM or the dictionary was dropped or + * altered then unpin the DSM segment. + * + * dict: key to search the dictionary's DSM segment. + * unpin_segment: true if we need to unpin the segment in case if the dictionary + * was dropped or altered. + */ +void +ts_dict_shmem_release(DictPointerData *dict, bool unpin_segment) +{ + TsearchDictKey key; + TsearchDictEntry *entry; + + /* + * If we didn't attach to a hash table then do nothing. + */ + if (!dict_table && !unpin_segment) + return; + /* + * But if we need to unpin the DSM segment to get of rid of the segment when + * the last interested process disconnects we need the hash table to find + * the dictionary's entry. + */ + else if (unpin_segment) + init_dict_table(); + + /* Set up key for hashtable search */ + key.db_id = MyDatabaseId; + key.dict = *dict; + + /* Try to find an entry in the hash table */ + entry = (TsearchDictEntry *) dshash_find(dict_table, &key, true); + + if (entry) + { + dsm_segment *seg; + + seg = dsm_find_mapping(entry->dict_dsm); + + if (seg) + { + dsm_unpin_mapping(seg); + dsm_detach(seg); + + entry->refcnt--; + } + + if (unpin_segment) + dsm_unpin_segment(entry->dict_dsm); + + if (entry->refcnt == 0) + { + /* + * Delete the entry iff there is no process which pinned mapping to + * the DSM segment. + */ + dshash_delete_entry(dict_table, entry); + } + else + dshash_release_lock(dict_table, entry); + } +} + +/* + * Allocate and initialize tsearch-related shared memory. + */ +void +TsearchShmemInit(void) +{ + bool found; + + tsearch_ctl = (TsearchCtlData *) + ShmemInitStruct("Full Text Search Ctl", sizeof(TsearchCtlData), &found); + + if (!found) + { + LWLockRegisterTranche(LWTRANCHE_TSEARCH_DSA, "tsearch_dsa"); + LWLockRegisterTranche(LWTRANCHE_TSEARCH_TABLE, "tsearch_table"); + + LWLockInitialize(&tsearch_ctl->lock, LWTRANCHE_TSEARCH_DSA); + + tsearch_ctl->area = DSM_HANDLE_INVALID; + tsearch_ctl->dict_table_handle = InvalidDsaPointer; + } +} + +/* + * Report shared memory space needed by TsearchShmemInit. + */ +Size +TsearchShmemSize(void) +{ + Size size = 0; + + /* size of service structure */ + size = add_size(size, MAXALIGN(sizeof(TsearchCtlData))); + + return size; +} + +/* + * A comparator function for TsearchDictKey. + * + * Returns 1 if keys are equal. + */ +static int +tsearch_dict_cmp(const void *a, const void *b, size_t size, void *arg) +{ + TsearchDictKey *k1 = (TsearchDictKey *) a; + TsearchDictKey *k2 = (TsearchDictKey *) b; + + if (k1->db_id == k2->db_id && k1->dict.id == k2->dict.id && + k1->dict.xmin == k2->dict.xmin && k1->dict.xmax == k2->dict.xmax && + ItemPointerEquals(&k1->dict.tid, &k2->dict.tid)) + return 0; + else + return 1; +} + +/* + * A hash function for TsearchDictKey. + */ +static uint32 +tsearch_dict_hash(const void *a, size_t size, void *arg) +{ + TsearchDictKey *k = (TsearchDictKey *) a; + uint32 s; + + s = hash_combine(0, hash_uint32(k->db_id)); + s = hash_combine(s, hash_uint32(k->dict.id)); + s = hash_combine(s, hash_uint32(k->dict.xmin)); + s = hash_combine(s, hash_uint32(k->dict.xmax)); + s = hash_combine(s, + hash_uint32(BlockIdGetBlockNumber(&k->dict.tid.ip_blkid))); + s = hash_combine(s, hash_uint32(k->dict.tid.ip_posid)); + + return s; +} + +/* + * Initialize hash table located in DSM. + * + * The hash table should be created and initialized if it doesn't exist yet. + */ +static void +init_dict_table(void) +{ + MemoryContext old_context; + dsa_area *dsa; + + /* Exit if hash table was initialized alread */ + if (dict_table) + return; + + old_context = MemoryContextSwitchTo(TopMemoryContext); + +recheck_table: + LWLockAcquire(&tsearch_ctl->lock, LW_SHARED); + + /* Hash table have been created already by someone */ + if (DsaPointerIsValid(tsearch_ctl->dict_table_handle)) + { + Assert(tsearch_ctl->area != DSM_HANDLE_INVALID); + + dsa = dsa_attach(tsearch_ctl->area); + + dict_table = dshash_attach(dsa, + &dict_table_params, + tsearch_ctl->dict_table_handle, + NULL); + } + else + { + /* Try to get exclusive lock */ + LWLockRelease(&tsearch_ctl->lock); + if (!LWLockAcquireOrWait(&tsearch_ctl->lock, LW_EXCLUSIVE)) + { + /* + * The lock was released by another backend and other backend + * has concurrently created the hash table already. + */ + goto recheck_table; + } + + dsa = dsa_create(LWTRANCHE_TSEARCH_DSA); + tsearch_ctl->area = dsa_get_handle(dsa); + + dict_table = dshash_create(dsa, &dict_table_params, NULL); + tsearch_ctl->dict_table_handle = dshash_get_hash_table_handle(dict_table); + + /* Remain attached until end of postmaster */ + dsa_pin(dsa); + } + + LWLockRelease(&tsearch_ctl->lock); + + /* Remain attached until end of session */ + dsa_pin_mapping(dsa); + + MemoryContextSwitchTo(old_context); +} diff --git a/src/backend/utils/cache/ts_cache.c b/src/backend/utils/cache/ts_cache.c index b2a0105ee8..57bc2bea5b 100644 --- a/src/backend/utils/cache/ts_cache.c +++ b/src/backend/utils/cache/ts_cache.c @@ -40,6 +40,7 @@ #include "commands/defrem.h" #include "tsearch/ts_cache.h" #include "tsearch/ts_public.h" +#include "tsearch/ts_shared.h" #include "utils/builtins.h" #include "utils/catcache.h" #include "utils/fmgroids.h" @@ -112,6 +113,36 @@ InvalidateTSCacheCallBack(Datum arg, int cacheid, uint32 hashvalue) TSCurrentConfigCache = InvalidOid; } +/* + * Unpin shared segments of all invalid dictionary entries. + */ +static void +flush_ts_dictionary_content(void) +{ + HASH_SEQ_STATUS status; + TSDictionaryCacheEntry *entry; + + if (!has_invalid_dictionary) + return; + + hash_seq_init(&status, TSDictionaryCacheHash); + while ((entry = (TSDictionaryCacheEntry *) hash_seq_search(&status)) != NULL) + { + if (!entry->isvalid) + { + DictPointerData dict_ptr; + + dict_ptr.id = entry->dictId; + dict_ptr.xmin = entry->dict_xmin; + dict_ptr.xmax = entry->dict_xmax; + dict_ptr.tid = entry->dict_tid; + ts_dict_shmem_release(&dict_ptr, false); + } + } + + has_invalid_dictionary = false; +} + /* * Fetch parser cache entry */ @@ -260,6 +291,13 @@ lookup_ts_dictionary_cache(Oid dictId) Form_pg_ts_template template; MemoryContext saveCtx; + /* + * It is possible that some invalid entries hold a DSM mapping and we + * need to unpin it to avoid memory leaking. We will unpin segments of + * all other invalid dictionaries. + */ + flush_ts_dictionary_content(); + tpdict = SearchSysCache1(TSDICTOID, ObjectIdGetDatum(dictId)); if (!HeapTupleIsValid(tpdict)) elog(ERROR, "cache lookup failed for text search dictionary %u", diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index c21bfe2f66..16b0858eda 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -219,6 +219,8 @@ typedef enum BuiltinTrancheIds LWTRANCHE_SHARED_TUPLESTORE, LWTRANCHE_TBM, LWTRANCHE_PARALLEL_APPEND, + LWTRANCHE_TSEARCH_DSA, + LWTRANCHE_TSEARCH_TABLE, LWTRANCHE_FIRST_USER_DEFINED } BuiltinTrancheIds; diff --git a/src/include/tsearch/ts_shared.h b/src/include/tsearch/ts_shared.h new file mode 100644 index 0000000000..b6d00bdc9e --- /dev/null +++ b/src/include/tsearch/ts_shared.h @@ -0,0 +1,26 @@ +/*------------------------------------------------------------------------- + * + * ts_shared.h + * tsearch shared memory management + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * + * src/include/tsearch/ts_shared.h + * + *------------------------------------------------------------------------- + */ +#ifndef TS_SHARED_H +#define TS_SHARED_H + +#include "tsearch/ts_public.h" + +typedef void *(*ts_dict_build_callback) (List *dictoptions, Size *size); + +extern void *ts_dict_shmem_location(DictInitData *init_data, + ts_dict_build_callback allocate_cb); +extern void ts_dict_shmem_release(DictPointerData *dict, bool unpin_segment); + +extern void TsearchShmemInit(void); +extern Size TsearchShmemSize(void); + +#endif /* TS_SHARED_H */