feat(search): add multilingual lexical fields

.github/workflows/ci.yml

@@ -167,6 +167,33 @@ jobs:
           version: "~> v2"
           args: release --snapshot --clean --skip=publish
 
+  kiwi-helper:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: tools/discrawl-kiwi/go.mod
+          cache-dependency-path: tools/discrawl-kiwi/go.sum
+
+      - name: Install Kiwi 0.23.2
+        working-directory: tools/discrawl-kiwi
+        run: bash install-kiwi.sh
+
+      - name: Test Go binding helper
+        working-directory: tools/discrawl-kiwi
+        run: go test -count=1 ./...
+
   secrets:
     runs-on: ubuntu-latest
     timeout-minutes: 15
@@ -192,3 +219,33 @@ jobs:
       - name: Scan working tree
         run: |
           "$(go env GOPATH)/bin/gitleaks" dir . --no-banner --redact
+
+  ja-helper:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: tools/discrawl-ja/go.mod
+          cache-dependency-path: tools/discrawl-ja/go.sum
+      - name: Test Japanese helper
+        working-directory: tools/discrawl-ja
+        run: go test -count=1 ./...
+
+  zh-helper:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+      - name: Setup Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: tools/discrawl-zh/go.mod
+          cache-dependency-path: tools/discrawl-zh/go.sum
+      - name: Test Chinese helper
+        working-directory: tools/discrawl-zh
+        run: go test -count=1 ./...

README.md

@@ -124,6 +124,7 @@ The full documentation lives at **[discrawl.sh](https://discrawl.sh/)**:
 - [Command reference](docs/README.md)
 - [Sync sources](docs/guides/sync-sources.md)
 - [Search modes](docs/guides/search-modes.md)
+- [Multilingual lexical benchmark](docs/benchmarks/multilingual-lexical.md)
 - [Git snapshot workflows](docs/guides/git-snapshots.md)
 - [Configuration](docs/configuration.md)
 

THIRD_PARTY_NOTICES.md

@@ -0,0 +1,39 @@
+# Third-party notices
+
+## Optional Korean lexical helper
+
+The separately built `tools/discrawl-kiwi` helper depends on:
+
+- [Kiwi](https://github.com/bab2min/Kiwi), copyright Minchul Lee,
+  licensed under GNU LGPL 2.1 or later.
+- [github.com/codingpot/kiwigo](https://github.com/codingpot/kiwigo), a Go
+  binding for Kiwi, licensed under GNU LGPL 2.1.
+
+These dependencies are optional and are not linked into the default Discrawl
+binary. Distributors who provide the helper or Kiwi native binaries must
+satisfy their applicable LGPL notice, source-access, and relinking
+requirements. Kiwi's full license text is available from its source
+repository and the GNU project:
+
+https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html
+
+## Optional Japanese lexical helper
+
+`tools/discrawl-ja` depends on:
+
+- [Kagome](https://github.com/ikawaha/kagome), MIT
+- [kagome-dict IPA](https://github.com/ikawaha/kagome-dict), MIT wrapper around
+  mecab-ipadic-2.7.0-20070801 / ICOT Free Software
+
+These dependencies are optional and are not linked into the default Discrawl
+binary. Preserve the IPADIC/ICOT notice when distributing the helper.
+
+## Optional Chinese lexical helper
+
+`tools/discrawl-zh` depends on [GSE](https://github.com/go-ego/gse), Apache-2.0.
+It is optional and is not linked into the default Discrawl binary.
+
+## In-process Arabic analyzer
+
+The Arabic light stemmer follows the Lucene/Bleve prefix-and-suffix contract.
+It is implemented in Discrawl itself and does not depend on Python.

docs/benchmarks/multilingual-lexical.md

@@ -0,0 +1,45 @@
+# Multilingual lexical benchmark
+
+This targeted microbenchmark checks the failure mode the optional multilingual
+fields are designed to fix: a query matching a useful subword inside an
+unsegmented Korean, Japanese, Chinese, or Arabic surface form.
+
+It is not a general relevance benchmark. The fixtures deliberately contain
+compound forms and attached Arabic proclitics that SQLite FTS5 `unicode61`
+cannot retrieve as independent terms.
+
+## Reproduce
+
+```bash
+# Korean helper: official Kiwi 0.23.2 + discrawl-kiwi
+# Japanese helper: go build ./tools/discrawl-ja
+# Chinese helper: go build ./tools/discrawl-zh
+# Arabic: in-process, no helper
+
+DISCRAWL_TOKENIZER_E2E=1 \
+DISCRAWL_KIWI_HELPER=/tmp/discrawl-kiwi \
+DISCRAWL_KIWI_MODEL=/tmp/kiwi-model/models/cong/base \
+DISCRAWL_JA_HELPER=/tmp/discrawl-ja \
+DISCRAWL_ZH_HELPER=/tmp/discrawl-zh \
+go test ./internal/store \
+  -run TestMultilingualLexicalQualityBenchmark \
+  -count=1 -v
+```
+
+## Result
+
+Measured on macOS arm64 with native Kiwi 0.23.2, Kagome Search, GSE CutSearch,
+and the in-process Arabic analyzer:
+
+| Language | `unicode61` recall@5 | Multilingual recall@5 |
+| --- | ---: | ---: |
+| Korean / Kiwi via kiwigo | 0/5 | 5/5 |
+| Japanese / Kagome Search | 0/5 | 5/5 |
+| Chinese / GSE search mode | 0/5 | 5/5 |
+| Arabic / in-process light stem | 0/5 | 5/5 |
+| **Total** | **0/20** | **20/20** |
+
+The benchmark therefore supports a narrow claim: configured language fields
+substantially improve recall for these segmentation cases. It does not claim a
+universal 100-point gain on natural Discord query distributions. No Python
+tokenizer is used.

docs/commands/search.md

@@ -18,7 +18,7 @@ discrawl --json search "websocket closed"
 
 ## Modes
 
-- `fts` (default) - SQLite FTS5 with `unicode61` tokenizer; newest matches first
+- `fts` (default) - SQLite FTS5 with `unicode61`; optional multilingual lexical fields use language-specific tokenizers and RRF
 - `semantic` - embeds the query, scores against locally stored vectors; errors out if embeddings are disabled or no compatible vectors exist
 - `hybrid` - runs both, deduplicates by message id, falls back to FTS when semantic is unavailable
 
@@ -36,6 +36,12 @@ discrawl --json search "websocket closed"
 
 User query terms are parameterized and quoted before `MATCH`, so tokens like `AND`, `OR`, `NOT`, `NEAR`, and `*` are searched as input terms instead of FTS operators. Punctuation still follows FTS5 tokenization rules.
 
+When `[search.lexical].languages` is non-empty, Discrawl searches the default
+field plus every configured language field, ranks each field independently,
+and merges the lists with reciprocal rank fusion. See
+[Search modes](../guides/search-modes.html#optional-multilingual-lexical-fields)
+for setup and dependency details.
+
 Ambiguous channel names fail with candidate guild/channel ids instead of
 silently searching multiple channels. Use `discrawl channels resolve <name>
 --json`, then keep the numeric id for repeatable workflows.

docs/configuration.md

@@ -77,6 +77,13 @@ full_cache = false
 [search]
 default_mode = "fts"
 
+[search.lexical]
+languages = [] # optional: "ko", "ja", "zh", "ar"
+kiwi_command = "discrawl-kiwi"
+kiwi_model = ""
+ja_command = "discrawl-ja"
+zh_command = "discrawl-zh"
+
 [search.embeddings]
 enabled = false
 provider = "openai"
@@ -140,6 +147,12 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and
 - `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion
 - `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement`
 - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time
+- `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi through `github.com/codingpot/kiwigo`), Japanese (`ja`, Kagome Search through `discrawl-ja`), Chinese (`zh`, GSE CutSearch through `discrawl-zh`), and Arabic (`ar`, in-process light stemming).
+- Korean, Japanese, and Chinese use separately built Go helpers so the default Discrawl binary stays small and CGO-free. Arabic is implemented in-process.
+- Optional lexical helpers are installed and built separately; Discrawl never downloads packages.
+- Tokenizer helpers load lazily on first indexing or search use. Commands that only inspect metadata do not start helpers.
+- Each enabled language adds an independent FTS5 table. Index and query text pass through the same tokenizer, and results from the default plus language-specific tables are merged with reciprocal rank fusion.
+- After adding or changing `search.lexical.languages`, run a writer command such as `discrawl sync` once so the configured lexical tables are built. Read-only commands never mutate the archive; new and edited messages update the tables automatically during later syncs.
 - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search
 - `[search.embeddings].dimensions` is an optional positive OpenAI projection size. Changing it requires `embed --rebuild` so stored message vectors and query vectors use the same dimensions.
 - `[search.embeddings].vector_backend` accepts `exact` or optional `turbovec`; turbovec requires Python plus the `turbovec` package and embedding dimensions divisible by 8.

docs/guides/search-modes.md

@@ -11,10 +11,60 @@
 ## FTS details
 
 - backed by SQLite FTS5 with the default `unicode61` tokenizer
+- optional `[search.lexical]` languages add independent tokenizer-specific FTS tables and merge their ranked results with reciprocal rank fusion
+- supported presets are Korean with native Kiwi through `kiwigo`, Japanese with Kagome Search, Chinese with GSE search mode, and Arabic with in-process light stemming
 - user query terms are parameterized and quoted before `MATCH`, so tokens like `AND`, `OR`, `NOT`, `NEAR`, and `*` are searched as input terms instead of FTS operators
 - punctuation still follows FTS5 tokenization rules
 - by default, `search` skips rows with no searchable content (attachment text, attachment filenames, embeds, and replies still count as content); use `--include-empty` to opt back in
 
+### Optional multilingual lexical fields
+
+Install Kiwi 0.23.2's dynamic library and base model, then build the Go helper:
+
+```bash
+git clone https://github.com/openclaw/discrawl
+cd discrawl/tools/discrawl-kiwi
+go build -o ~/.local/share/discrawl/bin/discrawl-kiwi .
+```
+
+`github.com/codingpot/kiwigo` links to the system Kiwi C API. Its upstream
+installation expects Kiwi headers and dynamic libraries under `/usr/local`;
+the model is the `kiwi_model_v0.23.2_base.tgz` release asset.
+
+Build the optional Japanese and Chinese helpers:
+
+```bash
+cd tools/discrawl-ja && go build -o ~/.local/share/discrawl/bin/discrawl-ja .
+cd ../discrawl-zh && go build -o ~/.local/share/discrawl/bin/discrawl-zh .
+```
+
+Configure the fields:
+
+```toml
+[search.lexical]
+languages = ["ko", "ja", "zh", "ar"]
+kiwi_command = "~/.local/share/discrawl/bin/discrawl-kiwi"
+kiwi_model = "~/.local/share/discrawl/models/kiwi/base"
+ja_command = "~/.local/share/discrawl/bin/discrawl-ja"
+zh_command = "~/.local/share/discrawl/bin/discrawl-zh"
+```
+
+Discrawl does not download or install helper packages; build the optional
+helpers separately and configure their absolute paths.
+
+Every message is analyzed into each configured field. This deliberately avoids
+language detection, so mixed-language Discord messages remain searchable
+through every enabled analyzer. Disk usage and indexing work increase with the
+number of fields; query-time RRF deduplicates message ids without mixing the
+different BM25 term statistics into one field.
+
+None of the lexical analyzers use Python. Korean, Japanese, and Chinese run as
+persistent Go helpers; Arabic is in-process. Helpers load lazily on first use
+and are not linked into the default Discrawl binary.
+
+See [Multilingual lexical benchmark](../benchmarks/multilingual-lexical.html)
+for the reproducible targeted quality check and its storage tradeoff.
+
 ## Semantic and hybrid prerequisites
 
 - `[search.embeddings]` configured in the Discrawl config file

internal/cli/cli.go

@@ -573,10 +573,7 @@ func (r *runtime) shouldAutoUpdateShare(mode shareUpdateMode) bool {
 
 func (r *runtime) autoUpdateShareIfLockAvailable(dbPath string, updateMode shareUpdateMode) error {
 	locked, err := r.tryWithSyncLock(func() error {
-		storeFactory := r.openStore
-		if storeFactory == nil {
-			storeFactory = store.Open
-		}
+		storeFactory := r.localStoreFactory()
 		var openErr error
 		r.store, openErr = storeFactory(r.ctx, dbPath)
 		if openErr != nil {
@@ -598,10 +595,7 @@ func (r *runtime) autoUpdateShareIfLockAvailable(dbPath string, updateMode share
 }
 
 func (r *runtime) openLocalStore(dbPath string, updateMode shareUpdateMode, fn func() error) error {
-	storeFactory := r.openStore
-	if storeFactory == nil {
-		storeFactory = store.Open
-	}
+	storeFactory := r.localStoreFactory()
 	var err error
 	r.store, err = storeFactory(r.ctx, dbPath)
 	if err != nil {
@@ -656,7 +650,7 @@ func (r *runtime) withExistingLocalStoreReadOnly(fn func() error) error {
 
 func (r *runtime) openLocalStoreReadOnly(dbPath string, fn func() error) error {
 	r.store = nil
-	s, err := store.OpenReadOnly(r.ctx, dbPath)
+	s, err := r.openConfiguredReadOnlyStore(dbPath)
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) {
 			return fn()
@@ -673,7 +667,7 @@ func (r *runtime) openLocalStoreReadOnly(dbPath string, fn func() error) error {
 
 func (r *runtime) openExistingLocalStoreReadOnly(dbPath string, fn func() error) error {
 	r.store = nil
-	s, err := store.OpenReadOnly(r.ctx, dbPath)
+	s, err := r.openConfiguredReadOnlyStore(dbPath)
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) {
 			return fn()
@@ -688,6 +682,19 @@ func (r *runtime) openExistingLocalStoreReadOnly(dbPath string, fn func() error)
 	return fn()
 }
 
+func (r *runtime) openConfiguredReadOnlyStore(path string) (*store.Store, error) {
+	if len(r.cfg.Search.Lexical.Languages) == 0 {
+		return store.OpenReadOnly(r.ctx, path)
+	}
+	return store.OpenReadOnlyWithOptions(r.ctx, path, store.OpenOptions{
+		LexicalLanguages:   r.cfg.Search.Lexical.Languages,
+		LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand,
+		LexicalKiwiModel:   r.cfg.Search.Lexical.KiwiModel,
+		LexicalJaCommand:   r.cfg.Search.Lexical.JaCommand,
+		LexicalZhCommand:   r.cfg.Search.Lexical.ZhCommand,
+	})
+}
+
 func (r *runtime) withServicesAuto(withDiscord, autoShareUpdate bool, fn func() error) error {
 	return r.withServicesAutoLocked(withDiscord, autoShareUpdate, false, fn)
 }
@@ -731,10 +738,7 @@ func (r *runtime) withServicesUpdateLockedOperation(withDiscord bool, updateMode
 }
 
 func (r *runtime) openServices(dbPath string, withDiscord bool, updateMode shareUpdateMode, fn func() error) error {
-	storeFactory := r.openStore
-	if storeFactory == nil {
-		storeFactory = store.Open
-	}
+	storeFactory := r.localStoreFactory()
 	var err error
 	r.store, err = storeFactory(r.ctx, dbPath)
 	if err != nil {
@@ -757,6 +761,21 @@ func (r *runtime) openServices(dbPath string, withDiscord bool, updateMode share
 	return fn()
 }
 
+func (r *runtime) localStoreFactory() func(context.Context, string) (*store.Store, error) {
+	if r.openStore != nil {
+		return r.openStore
+	}
+	return func(ctx context.Context, path string) (*store.Store, error) {
+		return store.OpenWithOptions(ctx, path, store.OpenOptions{
+			LexicalLanguages:   r.cfg.Search.Lexical.Languages,
+			LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand,
+			LexicalKiwiModel:   r.cfg.Search.Lexical.KiwiModel,
+			LexicalJaCommand:   r.cfg.Search.Lexical.JaCommand,
+			LexicalZhCommand:   r.cfg.Search.Lexical.ZhCommand,
+		})
+	}
+}
+
 func (r *runtime) ensureDiscordServices() error {
 	discordFactory := r.newDiscord
 	if discordFactory == nil {