Zhparser

https://github.com/amutu/zhparser

apt update
apt install gcc wget make git bzip2 postgresql-server-dev-14

cd /tmp
wget -q -O - <http://www.xunsearch.com/scws/down/scws-1.2.3.tar.bz2> | tar xf -
cd scws-1.2.3 
./configure 
make install

cd ..
git clone <https://github.com/amutu/zhparser.git>
make && make install

psql

CREATE EXTENSION zhparser;
CREATE TEXT SEARCH CONFIGURATION testzhcfg (PARSER = zhparser);
ALTER TEXT SEARCH CONFIGURATION testzhcfg ADD MAPPING FOR n,v,a,i,e,l WITH simple;
alter role all set zhparser.multi_short = on;
alter role all set zhparser.punctuation_ignore = on;

-- testing
SELECT * FROM ts_parse('zhparser', 'hello world! 2010年保障房建设在全国范围内获全面启动，从中央到地方纷纷加大 了保障房的建设和投入力度 。2011年，保障房进入了更大规模的建设阶段。住房城乡建设部党组书记、部长姜伟新去年底在全国住房城乡建设工作会议上表示，要继续推进保障性安居工程建设。');
SELECT to_tsvector('testzhcfg','“今年保障房新开工数量虽然有所下调，但实际的年度在建规模以及竣工规模会超以往年份，相对应的对资金的需求也会创历>史纪录。”陈国强说。在他看来，与2011年相比，2012年的保障房建设在资金配套上的压力将更为严峻。');
SELECT to_tsquery('testzhcfg', '保障房资金压力');

create table

-- dataset: <https://www.kaggle.com/datasets/ceshine/yet-another-chinese-news-dataset>

create table chinese_news
(
    title        text,
    description  text,
    image        text,
    url          text,
    source       text,
    publish_date text
);
create table chinese_news2 as (select * from chinese_news);

select count(*) from chinese_news;

-- Way-A: create gin index on column directly
create index idx_ts_description on chinese_news using gin (to_tsvector('zhcfg', description));

-- select
select description from chinese_news where to_tsvector(description) @@ to_tsquery('习近平');
explain analyse select description from chinese_news where to_tsvector(description) @@ to_tsquery('习近平');

-- insert value
insert into chinese_news(title, description, source, publish_date) values ('一果可爱捏', '别在这发骚，虽然但是一果可爱捏', 'fake news', '2022-07-12');
select description from chinese_news where to_tsvector(description) @@ to_tsquery('一果');

-- Way-B: create a separate full-text search column
alter table chinese_news2
    add column description_ts tsvector generated always as (to_tsvector('zhcfg',' ' || coalesce(description, ''))) stored;

create index idx_ts_description2 on chinese_news2 using GIN(description_ts);

-- select
select description from chinese_news2 where description_ts @@ to_tsquery('习近平');
explain analyse select description from chinese_news2 where description_ts @@ to_tsquery('习近平');

-- insert values
insert into chinese_news2(title, description, source, publish_date) values ('一果可爱捏', '别在这发骚，虽然但是一果可爱捏', 'fake news', '2022-07-12');
select description from chinese_news2 where description_ts @@ to_tsquery('一果');

-- compare with like
select description from chinese_news where description like '%习近平%';

-- expand zhparser
select * from ts_parse('zhparser', '一果可爱捏');
-- add new word
insert into zhparser.zhprs_custom_word values('一果');
select sync_zhprs_custom_word();
select * from ts_parse('zhparser', '一果可爱捏');

Pg_jieba

https://github.com/jaiminpan/pg_jieba

apt update
apt install gcc wget make git cmake build-essential postgresql-server-dev-14

git clone --depth 1 <https://github.com/jaiminpan/pg_jieba.git>
cd pg_jieba
git submodule update --init --recursive

mkdir build
cd build
cmake ..
make
make install

git clone <https://github.com/yanyiwu/cppjieba.git>

pgsql

-- create extension
create extension pg_jieba;
-- testing
select * from ts_token_type('jieba');
select * from to_tsquery('jiebacfg', '小明硕士毕业于中国科学院计算所，后在日本京都大学深造');
select * from to_tsvector('jiebacfg', '小明硕士毕业于中国科学院计算所，后在日本京都大学深造');
select * from ts_debug('jiebacfg', '小明硕士毕业于中国科学院计算所，后在日本京都大学深造');

-- testing different config
select * from to_tsvector('jiebamp', '小明硕士毕业于中国科学院计算所，后在日本京都大学深造');
select * from to_tsvector('jiebahmm', '小明硕士毕业于中国科学院计算所，后在日本京都大学深造');
select * from to_tsvector('jiebacfg', '小明硕士毕业于中国科学院计算所，后在日本京都大学深造');
select * from to_tsvector('jiebaqry', '小明硕士毕业于中国科学院计算所，后在日本京都大学深造');

-- dataset: <https://www.kaggle.com/datasets/ceshine/yet-another-chinese-news-dataset>
create table chinese_news
(
    title        text,
    description  text,
    image        text,
    url          text,
    source       text,
    publish_date text
);

-- directly select
select description from chinese_news where to_tsvector('jiebaqry', coalesce(description, '')) @@ to_tsquery('一果');

-- create gin index
create index idx_description_ts on chinese_news using gin (to_tsvector('jiebaqry', coalesce(description, '')));

-- select via gin index
select description from chinese_news where to_tsvector('jiebaqry', coalesce(description, '')) @@ to_tsquery('一果');
select description from chinese_news where to_tsvector('jiebaqry', coalesce(description, '')) @@ to_tsquery('一果 & 可爱');
select description from chinese_news where to_tsvector('jiebaqry', coalesce(description, '')) @@ to_tsquery('一果 | 可爱');

-- custom dict: /usr/share/postgresql/14/tsearch_data/jieba_user.dict
select jieba_reload_dict();
reindex index concurrently idx_description_ts;
select description from chinese_news where to_tsvector('jiebaqry', coalesce(description, '')) @@ to_tsquery('许章润');