diff --git a/doc/src/sgml/textsearch.sgml b/doc/src/sgml/textsearch.sgml index e1829277d0..c534487743 100644 --- a/doc/src/sgml/textsearch.sgml +++ b/doc/src/sgml/textsearch.sgml @@ -2828,6 +2828,7 @@ iconv -f ISO_8859-1 -t UTF-8 -o nn_no.dict nn_NO.dic CREATE TEXT SEARCH DICTIONARY english_hunspell ( TEMPLATE = ispell, + Shareable = false, DictFile = en_us, AffFile = en_us, Stopwords = english); @@ -2842,6 +2843,9 @@ CREATE TEXT SEARCH DICTIONARY english_hunspell ( The stop-words file has the same format explained above for the simple dictionary type. The format of the other files is not specified here but is available from the above-mentioned web sites. + Shareable controls loading into shared memory. By + default it is true (see more in + ). @@ -3036,7 +3040,8 @@ CREATE TEXT SEARCH DICTIONARY english_stem ( Some dictionaries, especially Ispell, consumes a noticable value of memory. Size of a dictionary can reach tens of megabytes. Most of them also stores configuration in text files. A dictionary is compiled - during first access per a user session. + during first access per a user session. Currently only + Ispell supports loading into shared memory. diff --git a/src/backend/tsearch/dict_ispell.c b/src/backend/tsearch/dict_ispell.c index 60ef770dbd..5384f7d87a 100644 --- a/src/backend/tsearch/dict_ispell.c +++ b/src/backend/tsearch/dict_ispell.c @@ -38,7 +38,8 @@ typedef struct } DictISpell; static void parse_dictoptions(List *dictoptions, - char **dictfile, char **afffile, char **stopfile); + char **dictfile, char **afffile, char **stopfile, + bool *isshared); static void *dispell_build(List *dictoptions, Size *size); Datum @@ -49,15 +50,22 @@ dispell_init(PG_FUNCTION_ARGS) DictISpell *d; void *dict_location; char *stopfile; + bool isshared; d = (DictISpell *) palloc0(sizeof(DictISpell)); - parse_dictoptions(dictoptions, NULL, NULL, &stopfile); + parse_dictoptions(dictoptions, NULL, NULL, &stopfile, &isshared); + /* Make stop word list */ if (stopfile) readstoplist(stopfile, &(d->stoplist), lowerstr); - dict_location = ts_dict_shmem_location(dictid, dictoptions, dispell_build); + /* Make or get from shared memory dictionary itself */ + if (isshared) + dict_location = ts_dict_shmem_location(dictid, dictoptions, dispell_build); + else + dict_location = dispell_build(dictoptions, NULL); + Assert(dict_location); d->obj.dict = (IspellDictData *) dict_location; @@ -111,9 +119,10 @@ dispell_lexize(PG_FUNCTION_ARGS) static void parse_dictoptions(List *dictoptions, char **dictfile, char **afffile, - char **stopfile) + char **stopfile, bool *isshared) { ListCell *l; + bool isshared_defined = false; if (dictfile) *dictfile = NULL; @@ -121,6 +130,8 @@ parse_dictoptions(List *dictoptions, char **dictfile, char **afffile, *afffile = NULL; if (stopfile) *stopfile = NULL; + if (isshared) + *isshared = true; foreach(l, dictoptions) { @@ -159,6 +170,19 @@ parse_dictoptions(List *dictoptions, char **dictfile, char **afffile, errmsg("multiple StopWords parameters"))); *stopfile = defGetString(defel); } + else if (pg_strcasecmp(defel->defname, "Shareable") == 0) + { + if (!isshared) + continue; + + if (isshared_defined) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple Shareable parameters"))); + + *isshared = defGetBoolean(defel); + isshared_defined = true; + } else { ereport(ERROR, @@ -181,7 +205,7 @@ dispell_build(List *dictoptions, Size *size) char *dictfile, *afffile; - parse_dictoptions(dictoptions, &dictfile, &afffile, NULL); + parse_dictoptions(dictoptions, &dictfile, &afffile, NULL, NULL); if (!afffile) { @@ -213,6 +237,7 @@ dispell_build(List *dictoptions, Size *size) NIFinishBuild(&build); /* Return the buffer and its size */ - *size = build.dict_size; + if (size) + *size = build.dict_size; return build.dict; } diff --git a/src/test/regress/expected/tsdicts.out b/src/test/regress/expected/tsdicts.out index 0744ef803b..932d75acec 100644 --- a/src/test/regress/expected/tsdicts.out +++ b/src/test/regress/expected/tsdicts.out @@ -194,6 +194,7 @@ SELECT ts_lexize('hunspell', 'footballyklubber'); -- Test ISpell dictionary with hunspell affix file with FLAG long parameter CREATE TEXT SEARCH DICTIONARY hunspell_long ( Template=ispell, + Shareable=false, DictFile=hunspell_sample_long, AffFile=hunspell_sample_long ); @@ -290,6 +291,7 @@ SELECT ts_lexize('hunspell_long', 'footballyklubber'); -- Test ISpell dictionary with hunspell affix file with FLAG num parameter CREATE TEXT SEARCH DICTIONARY hunspell_num ( Template=ispell, + Shareable=false, DictFile=hunspell_sample_num, AffFile=hunspell_sample_num ); @@ -580,3 +582,58 @@ SELECT to_tsvector('thesaurus_tst', 'Booking tickets is looking like a booking a 'card':3,10 'invit':2,9 'like':6 'look':5 'order':1,8 (1 row) +-- Test shared dictionaries +CREATE TEXT SEARCH DICTIONARY shared_ispell ( + Template=ispell, + DictFile=ispell_sample, + AffFile=ispell_sample +); +-- Make sure that dictionaries in shared memory +SELECT ts_lexize('ispell', 'skies'); + ts_lexize +----------- + {sky} +(1 row) + +SELECT ts_lexize('hunspell', 'skies'); + ts_lexize +----------- + {sky} +(1 row) + +SELECT ts_lexize('shared_ispell', 'skies'); + ts_lexize +----------- + {sky} +(1 row) + +SELECT schemaname, dictname FROM pg_ts_shared_dictionaries; + schemaname | dictname +------------+--------------- + public | ispell + public | hunspell + public | shared_ispell +(3 rows) + +-- shared_ispell space should be released in shared memory +DROP TEXT SEARCH DICTIONARY shared_ispell; +-- Make sure that dictionaries in shared memory, DROP invalidates cache +SELECT ts_lexize('ispell', 'skies'); + ts_lexize +----------- + {sky} +(1 row) + +SELECT ts_lexize('hunspell', 'skies'); + ts_lexize +----------- + {sky} +(1 row) + +SELECT schemaname, dictname FROM pg_ts_shared_dictionaries; + schemaname | dictname +------------+---------- + public | ispell + public | hunspell +(2 rows) + diff --git a/src/test/regress/sql/tsdicts.sql b/src/test/regress/sql/tsdicts.sql index a5a569e1ad..04c1161141 100644 --- a/src/test/regress/sql/tsdicts.sql +++ b/src/test/regress/sql/tsdicts.sql @@ -51,6 +51,7 @@ SELECT ts_lexize('hunspell', 'footballyklubber'); -- Test ISpell dictionary with hunspell affix file with FLAG long parameter CREATE TEXT SEARCH DICTIONARY hunspell_long ( Template=ispell, + Shareable=false, DictFile=hunspell_sample_long, AffFile=hunspell_sample_long ); @@ -75,6 +76,7 @@ SELECT ts_lexize('hunspell_long', 'footballyklubber'); -- Test ISpell dictionary with hunspell affix file with FLAG num parameter CREATE TEXT SEARCH DICTIONARY hunspell_num ( Template=ispell, + Shareable=false, DictFile=hunspell_sample_num, AffFile=hunspell_sample_num ); @@ -188,3 +190,26 @@ ALTER TEXT SEARCH CONFIGURATION thesaurus_tst ALTER MAPPING FOR SELECT to_tsvector('thesaurus_tst', 'one postgres one two one two three one'); SELECT to_tsvector('thesaurus_tst', 'Supernovae star is very new star and usually called supernovae (abbreviation SN)'); SELECT to_tsvector('thesaurus_tst', 'Booking tickets is looking like a booking a tickets'); + +-- Test shared dictionaries +CREATE TEXT SEARCH DICTIONARY shared_ispell ( + Template=ispell, + DictFile=ispell_sample, + AffFile=ispell_sample +); + +-- Make sure that dictionaries in shared memory +SELECT ts_lexize('ispell', 'skies'); +SELECT ts_lexize('hunspell', 'skies'); +SELECT ts_lexize('shared_ispell', 'skies'); + +SELECT schemaname, dictname FROM pg_ts_shared_dictionaries; + +-- shared_ispell space should be released in shared memory +DROP TEXT SEARCH DICTIONARY shared_ispell; + +-- Make sure that dictionaries in shared memory, DROP invalidates cache +SELECT ts_lexize('ispell', 'skies'); +SELECT ts_lexize('hunspell', 'skies'); + +SELECT schemaname, dictname FROM pg_ts_shared_dictionaries;