From 4b41ae9ea73efb96ba6c66724ad97c7f52aab85d Mon Sep 17 00:00:00 2001 From: "Jeffrey (Dongkyu) Kim" Date: Tue, 18 Aug 2026 17:27:20 +0900 Subject: [PATCH 1/5] feat(search): add multilingual lexical fields --- README.md | 1 + docs/benchmarks/multilingual-lexical.md | 48 +++ docs/commands/search.md | 8 +- docs/configuration.md | 8 + docs/guides/search-modes.md | 29 ++ internal/cli/cli.go | 41 ++- internal/config/config.go | 35 +- internal/config/config_test.go | 37 +++ internal/store/lexical.go | 266 +++++++++++++++ internal/store/lexical_python.go | 312 ++++++++++++++++++ internal/store/lexical_python_command_unix.go | 24 ++ .../store/lexical_python_command_windows.go | 28 ++ internal/store/lexical_python_test.go | 143 ++++++++ internal/store/lexical_search.go | 150 +++++++++ internal/store/multilingual_benchmark_test.go | 131 ++++++++ internal/store/multilingual_edges_test.go | 101 ++++++ internal/store/multilingual_lifecycle_test.go | 145 ++++++++ .../store/multilingual_python_e2e_test.go | 52 +++ internal/store/multilingual_search_test.go | 148 +++++++++ internal/store/query.go | 3 + internal/store/store.go | 59 +++- internal/store/write.go | 36 +- 22 files changed, 1769 insertions(+), 36 deletions(-) create mode 100644 docs/benchmarks/multilingual-lexical.md create mode 100644 internal/store/lexical.go create mode 100644 internal/store/lexical_python.go create mode 100644 internal/store/lexical_python_command_unix.go create mode 100644 internal/store/lexical_python_command_windows.go create mode 100644 internal/store/lexical_python_test.go create mode 100644 internal/store/lexical_search.go create mode 100644 internal/store/multilingual_benchmark_test.go create mode 100644 internal/store/multilingual_edges_test.go create mode 100644 internal/store/multilingual_lifecycle_test.go create mode 100644 internal/store/multilingual_python_e2e_test.go create mode 100644 internal/store/multilingual_search_test.go diff --git a/README.md b/README.md index 1837d5cf..756f8f33 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ The full documentation lives at **[discrawl.sh](https://discrawl.sh/)**: - [Command reference](docs/README.md) - [Sync sources](docs/guides/sync-sources.md) - [Search modes](docs/guides/search-modes.md) +- [Multilingual lexical benchmark](docs/benchmarks/multilingual-lexical.md) - [Git snapshot workflows](docs/guides/git-snapshots.md) - [Configuration](docs/configuration.md) diff --git a/docs/benchmarks/multilingual-lexical.md b/docs/benchmarks/multilingual-lexical.md new file mode 100644 index 00000000..d6aac50a --- /dev/null +++ b/docs/benchmarks/multilingual-lexical.md @@ -0,0 +1,48 @@ +# Multilingual lexical benchmark + +This targeted microbenchmark checks the failure mode the optional multilingual +fields are designed to fix: a query matching a useful subword inside an +unsegmented Korean, Japanese, Chinese, or Arabic surface form. + +It is not a general relevance benchmark. The fixtures deliberately contain +compound forms and attached Arabic proclitics that SQLite FTS5 `unicode61` +cannot retrieve as independent terms. + +## Reproduce + +```bash +python3 -m venv /tmp/discrawl-tokenizer-e2e +/tmp/discrawl-tokenizer-e2e/bin/python -m pip install \ + kiwipiepy==0.23.2 \ + sudachipy==0.6.11 \ + sudachidict_core==20260723 \ + jieba==0.42.1 \ + snowballstemmer==3.1.1 + +DISCRAWL_TOKENIZER_E2E=1 \ +DISCRAWL_TOKENIZER_PYTHON=/tmp/discrawl-tokenizer-e2e/bin/python \ +go test ./internal/store \ + -run TestMultilingualLexicalQualityBenchmark \ + -count=1 -v +``` + +## Result + +Measured on macOS arm64 with Python 3.13.2: + +| Language | `unicode61` recall@5 | Multilingual recall@5 | +| --- | ---: | ---: | +| Korean / Kiwi | 0/5 | 5/5 | +| Japanese / Sudachi A | 0/5 | 5/5 | +| Chinese / Jieba search mode | 0/5 | 5/5 | +| Arabic / Snowball + proclitics | 0/5 | 5/5 | +| **Total** | **0/20** | **20/20** | + +Across repeated runs of this 20-message fixture, the database with four extra +FTS tables used 1.36-1.37x the SQLite pages of the baseline. Real corpora will +have different ratios because base tables, attachments, and metadata are not +duplicated while postings scale with enabled analyzers. + +The benchmark therefore supports a narrow claim: configured language fields +substantially improve recall for these segmentation cases. It does not claim a +universal 100-point gain on natural Discord query distributions. diff --git a/docs/commands/search.md b/docs/commands/search.md index 10466eef..afcd849a 100644 --- a/docs/commands/search.md +++ b/docs/commands/search.md @@ -18,7 +18,7 @@ discrawl --json search "websocket closed" ## Modes -- `fts` (default) - SQLite FTS5 with `unicode61` tokenizer; newest matches first +- `fts` (default) - SQLite FTS5 with `unicode61`; optional multilingual lexical fields use language-specific tokenizers and RRF - `semantic` - embeds the query, scores against locally stored vectors; errors out if embeddings are disabled or no compatible vectors exist - `hybrid` - runs both, deduplicates by message id, falls back to FTS when semantic is unavailable @@ -36,6 +36,12 @@ discrawl --json search "websocket closed" User query terms are parameterized and quoted before `MATCH`, so tokens like `AND`, `OR`, `NOT`, `NEAR`, and `*` are searched as input terms instead of FTS operators. Punctuation still follows FTS5 tokenization rules. +When `[search.lexical].languages` is non-empty, Discrawl searches the default +field plus every configured language field, ranks each field independently, +and merges the lists with reciprocal rank fusion. See +[Search modes](../guides/search-modes.html#optional-multilingual-lexical-fields) +for setup and dependency details. + Ambiguous channel names fail with candidate guild/channel ids instead of silently searching multiple channels. Use `discrawl channels resolve --json`, then keep the numeric id for repeatable workflows. diff --git a/docs/configuration.md b/docs/configuration.md index c894186b..85f638ef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -77,6 +77,10 @@ full_cache = false [search] default_mode = "fts" +[search.lexical] +languages = [] # optional: "ko", "ja", "zh", "ar" +python = "python3" + [search.embeddings] enabled = false provider = "openai" @@ -140,6 +144,10 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and - `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion - `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement` - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time +- `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi), Japanese (`ja`, Sudachi), Chinese (`zh`, Jieba), and Arabic (`ar`, Snowball plus proclitic splitting). +- Multilingual lexical search requires the configured Python interpreter to provide `kiwipiepy`, `sudachipy`, `sudachidict_core`, `jieba`, and `snowballstemmer` for the enabled languages. Discrawl reports the missing module while opening the archive instead of silently falling back. +- Each enabled language adds an independent FTS5 table. Index and query text pass through the same tokenizer, and results from the default plus language-specific tables are merged with reciprocal rank fusion. +- After adding or changing `search.lexical.languages`, run a writer command such as `discrawl sync` once so the configured lexical tables are built. Read-only commands never mutate the archive; new and edited messages update the tables automatically during later syncs. - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search - `[search.embeddings].dimensions` is an optional positive OpenAI projection size. Changing it requires `embed --rebuild` so stored message vectors and query vectors use the same dimensions. - `[search.embeddings].vector_backend` accepts `exact` or optional `turbovec`; turbovec requires Python plus the `turbovec` package and embedding dimensions divisible by 8. diff --git a/docs/guides/search-modes.md b/docs/guides/search-modes.md index a251bea5..e70f0813 100644 --- a/docs/guides/search-modes.md +++ b/docs/guides/search-modes.md @@ -11,10 +11,39 @@ ## FTS details - backed by SQLite FTS5 with the default `unicode61` tokenizer +- optional `[search.lexical]` languages add independent tokenizer-specific FTS tables and merge their ranked results with reciprocal rank fusion +- supported presets are Korean with Kiwi, Japanese with Sudachi A-mode, Chinese with Jieba search mode, and Arabic with Snowball stemming plus proclitic splitting - user query terms are parameterized and quoted before `MATCH`, so tokens like `AND`, `OR`, `NOT`, `NEAR`, and `*` are searched as input terms instead of FTS operators - punctuation still follows FTS5 tokenization rules - by default, `search` skips rows with no searchable content (attachment text, attachment filenames, embeds, and replies still count as content); use `--include-empty` to opt back in +### Optional multilingual lexical fields + +Create an isolated Python environment and install only the tokenizer presets you enable: + +```bash +python3 -m venv ~/.local/share/discrawl/tokenizers +~/.local/share/discrawl/tokenizers/bin/python -m pip install \ + kiwipiepy sudachipy sudachidict_core jieba snowballstemmer +``` + +Then configure the fields: + +```toml +[search.lexical] +languages = ["ko", "ja", "zh", "ar"] +python = "~/.local/share/discrawl/tokenizers/bin/python" # ~ is expanded +``` + +Every message is analyzed into each configured field. This deliberately avoids +language detection, so mixed-language Discord messages remain searchable +through every enabled analyzer. Disk usage and indexing work increase with the +number of fields; query-time RRF deduplicates message ids without mixing the +different BM25 term statistics into one field. + +See [Multilingual lexical benchmark](../benchmarks/multilingual-lexical.html) +for the reproducible targeted quality check and its storage tradeoff. + ## Semantic and hybrid prerequisites - `[search.embeddings]` configured in the Discrawl config file diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 2c30dcc9..68c7680d 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -573,10 +573,7 @@ func (r *runtime) shouldAutoUpdateShare(mode shareUpdateMode) bool { func (r *runtime) autoUpdateShareIfLockAvailable(dbPath string, updateMode shareUpdateMode) error { locked, err := r.tryWithSyncLock(func() error { - storeFactory := r.openStore - if storeFactory == nil { - storeFactory = store.Open - } + storeFactory := r.localStoreFactory() var openErr error r.store, openErr = storeFactory(r.ctx, dbPath) if openErr != nil { @@ -598,10 +595,7 @@ func (r *runtime) autoUpdateShareIfLockAvailable(dbPath string, updateMode share } func (r *runtime) openLocalStore(dbPath string, updateMode shareUpdateMode, fn func() error) error { - storeFactory := r.openStore - if storeFactory == nil { - storeFactory = store.Open - } + storeFactory := r.localStoreFactory() var err error r.store, err = storeFactory(r.ctx, dbPath) if err != nil { @@ -656,7 +650,7 @@ func (r *runtime) withExistingLocalStoreReadOnly(fn func() error) error { func (r *runtime) openLocalStoreReadOnly(dbPath string, fn func() error) error { r.store = nil - s, err := store.OpenReadOnly(r.ctx, dbPath) + s, err := r.openConfiguredReadOnlyStore(dbPath) if err != nil { if errors.Is(err, os.ErrNotExist) { return fn() @@ -673,7 +667,7 @@ func (r *runtime) openLocalStoreReadOnly(dbPath string, fn func() error) error { func (r *runtime) openExistingLocalStoreReadOnly(dbPath string, fn func() error) error { r.store = nil - s, err := store.OpenReadOnly(r.ctx, dbPath) + s, err := r.openConfiguredReadOnlyStore(dbPath) if err != nil { if errors.Is(err, os.ErrNotExist) { return fn() @@ -688,6 +682,16 @@ func (r *runtime) openExistingLocalStoreReadOnly(dbPath string, fn func() error) return fn() } +func (r *runtime) openConfiguredReadOnlyStore(path string) (*store.Store, error) { + if len(r.cfg.Search.Lexical.Languages) == 0 { + return store.OpenReadOnly(r.ctx, path) + } + return store.OpenReadOnlyWithOptions(r.ctx, path, store.OpenOptions{ + LexicalLanguages: r.cfg.Search.Lexical.Languages, + LexicalPython: r.cfg.Search.Lexical.Python, + }) +} + func (r *runtime) withServicesAuto(withDiscord, autoShareUpdate bool, fn func() error) error { return r.withServicesAutoLocked(withDiscord, autoShareUpdate, false, fn) } @@ -731,10 +735,7 @@ func (r *runtime) withServicesUpdateLockedOperation(withDiscord bool, updateMode } func (r *runtime) openServices(dbPath string, withDiscord bool, updateMode shareUpdateMode, fn func() error) error { - storeFactory := r.openStore - if storeFactory == nil { - storeFactory = store.Open - } + storeFactory := r.localStoreFactory() var err error r.store, err = storeFactory(r.ctx, dbPath) if err != nil { @@ -757,6 +758,18 @@ func (r *runtime) openServices(dbPath string, withDiscord bool, updateMode share return fn() } +func (r *runtime) localStoreFactory() func(context.Context, string) (*store.Store, error) { + if r.openStore != nil { + return r.openStore + } + return func(ctx context.Context, path string) (*store.Store, error) { + return store.OpenWithOptions(ctx, path, store.OpenOptions{ + LexicalLanguages: r.cfg.Search.Lexical.Languages, + LexicalPython: r.cfg.Search.Lexical.Python, + }) + } +} + func (r *runtime) ensureDiscordServices() error { discordFactory := r.newDiscord if discordFactory == nil { diff --git a/internal/config/config.go b/internal/config/config.go index d9caa565..95390991 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -67,8 +67,14 @@ type SyncConfig struct { } type SearchConfig struct { - DefaultMode string `toml:"default_mode"` - Embeddings EmbeddingsConfig `toml:"embeddings"` + DefaultMode string `toml:"default_mode"` + Lexical LexicalSearchConfig `toml:"lexical"` + Embeddings EmbeddingsConfig `toml:"embeddings"` +} + +type LexicalSearchConfig struct { + Languages []string `toml:"languages,omitempty"` + Python string `toml:"python"` } type ShareConfig struct { @@ -154,6 +160,9 @@ func Default() Config { }, Search: SearchConfig{ DefaultMode: "fts", + Lexical: LexicalSearchConfig{ + Python: "python3", + }, Embeddings: EmbeddingsConfig{ Enabled: false, Provider: "openai", @@ -300,6 +309,28 @@ func (c *Config) Normalize() error { if c.Search.DefaultMode == "" { c.Search.DefaultMode = "fts" } + c.Search.Lexical.Python = strings.TrimSpace(c.Search.Lexical.Python) + if c.Search.Lexical.Python == "" { + c.Search.Lexical.Python = "python3" + } + seenLexicalLanguages := make(map[string]struct{}, len(c.Search.Lexical.Languages)) + normalizedLexicalLanguages := make([]string, 0, len(c.Search.Lexical.Languages)) + for _, language := range c.Search.Lexical.Languages { + language = strings.ToLower(strings.TrimSpace(language)) + switch language { + case "ko", "ja", "zh", "ar": + case "": + continue + default: + return fmt.Errorf("unsupported search.lexical language %q; use ko, ja, zh, or ar", language) + } + if _, ok := seenLexicalLanguages[language]; ok { + return fmt.Errorf("duplicate search.lexical language %q", language) + } + seenLexicalLanguages[language] = struct{}{} + normalizedLexicalLanguages = append(normalizedLexicalLanguages, language) + } + c.Search.Lexical.Languages = normalizedLexicalLanguages c.Search.Embeddings.Provider = strings.ToLower(strings.TrimSpace(c.Search.Embeddings.Provider)) c.Search.Embeddings.Model = strings.TrimSpace(c.Search.Embeddings.Model) c.Search.Embeddings.BaseURL = strings.TrimRight(strings.TrimSpace(c.Search.Embeddings.BaseURL), "/") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d6bbffff..4562ca83 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -249,6 +249,43 @@ func TestDefaultSyncConcurrencyBounds(t *testing.T) { require.Equal(t, 32, defaultSyncConcurrency()) } +func TestLoadConfigMultilingualLexicalSearch(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(path, []byte(` +version = 1 + +[discord] +token_source = "env" + +[search.lexical] +languages = ["ko", "ja", "zh", "ar"] +python = "/opt/discrawl-tokenizers/bin/python" +`), 0o600)) + + cfg, err := Load(path) + require.NoError(t, err) + require.Equal(t, []string{"ko", "ja", "zh", "ar"}, cfg.Search.Lexical.Languages) + require.Equal(t, "/opt/discrawl-tokenizers/bin/python", cfg.Search.Lexical.Python) +} + +func TestNormalizeRejectsUnsupportedLexicalLanguage(t *testing.T) { + t.Parallel() + + cfg := Default() + cfg.Search.Lexical.Languages = []string{"ko", "klingon"} + require.ErrorContains(t, cfg.Normalize(), `unsupported search.lexical language "klingon"`) +} + +func TestNormalizeRejectsDuplicateLexicalLanguages(t *testing.T) { + t.Parallel() + + cfg := Default() + cfg.Search.Lexical.Languages = []string{"ko", "ko"} + require.ErrorContains(t, cfg.Normalize(), `duplicate search.lexical language "ko"`) +} + func TestResolveDiscordTokenFromEnv(t *testing.T) { cfg := Default() t.Setenv(DefaultTokenEnv, "Bot env-token") diff --git a/internal/store/lexical.go b/internal/store/lexical.go new file mode 100644 index 00000000..25b8eae3 --- /dev/null +++ b/internal/store/lexical.go @@ -0,0 +1,266 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" + "time" +) + +const lexicalFTSVersion = "1" + +type LexicalTokenizer interface { + Tokenize(context.Context, string) (string, error) + Close() error +} + +func openWithLexicalTokenizers( + ctx context.Context, + path string, + tokenizers map[string]LexicalTokenizer, +) (*Store, error) { + base, err := openBaseStore(ctx, path) + if err != nil { + closeLexicalTokenizers(tokenizers) + return nil, err + } + store := &Store{ + db: base.DB(), + q: newStoreQueries(base.DB()), + path: path, + baseClose: base.Close, + lexicalTokenizers: tokenizers, + } + if err := store.migrate(ctx); err != nil { + _ = store.Close() + return nil, err + } + if err := store.ensureLexicalFTS(ctx); err != nil { + _ = store.Close() + return nil, err + } + return store, nil +} + +func (s *Store) lexicalLanguages() []string { + languages := make([]string, 0, len(s.lexicalTokenizers)) + for language := range s.lexicalTokenizers { + languages = append(languages, language) + } + sort.Strings(languages) + return languages +} + +func (s *Store) tokenizeLexical(ctx context.Context, text string) (map[string]string, error) { + if len(s.lexicalTokenizers) == 0 || strings.TrimSpace(text) == "" { + return nil, nil + } + tokenized := make(map[string]string, len(s.lexicalTokenizers)) + for _, language := range s.lexicalLanguages() { + content, err := s.lexicalTokenizers[language].Tokenize(ctx, text) + if err != nil { + return nil, fmt.Errorf("tokenize %s text: %w", language, err) + } + tokenized[language] = content + } + return tokenized, nil +} + +func (s *Store) ensureLexicalFTS(ctx context.Context) error { + for _, language := range s.lexicalLanguages() { + var version sql.NullString + err := s.db.QueryRowContext(ctx, ` + select cursor from sync_state where scope = ? + `, lexicalFTSScope(language)).Scan(&version) + if err == nil && version.String == lexicalFTSVersion { + continue + } + if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("check %s lexical index version: %w", language, err) + } + if err := s.rebuildLexicalFTS(ctx, language); err != nil { + return err + } + if _, err := s.db.ExecContext(ctx, ` + insert into sync_state(scope, cursor, updated_at) + values(?, ?, ?) + on conflict(scope) do update set + cursor = excluded.cursor, + updated_at = excluded.updated_at + `, lexicalFTSScope(language), lexicalFTSVersion, time.Now().UTC().Format(timeLayout)); err != nil { + return fmt.Errorf("stamp %s lexical index version: %w", language, err) + } + } + return nil +} + +func (s *Store) rebuildLexicalIndexes(ctx context.Context) error { + for _, language := range s.lexicalLanguages() { + if err := s.rebuildLexicalFTS(ctx, language); err != nil { + return err + } + } + return nil +} + +func (s *Store) rebuildLexicalFTS(ctx context.Context, language string) error { + table := lexicalFTSTable(language) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer rollback(tx) + if _, err := tx.ExecContext(ctx, "drop table if exists "+table); err != nil { + return fmt.Errorf("drop %s: %w", table, err) + } + if _, err := tx.ExecContext(ctx, createLexicalFTSSQL(table)); err != nil { + return fmt.Errorf("create %s: %w", table, err) + } + if err := configureFTSBulkLoad(ctx, tx, table); err != nil { + return err + } + rows, err := tx.QueryContext(ctx, lexicalRebuildRowsSQL) + if err != nil { + return fmt.Errorf("query %s rebuild rows: %w", table, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var message MessageRecord + if err := rows.Scan( + &message.ID, + &message.GuildID, + &message.ChannelID, + &message.AuthorID, + &message.AuthorName, + &message.ChannelName, + &message.NormalizedContent, + ); err != nil { + return fmt.Errorf("scan %s rebuild row: %w", table, err) + } + content, err := s.lexicalTokenizers[language].Tokenize(ctx, message.NormalizedContent) + if err != nil { + return fmt.Errorf("tokenize %s rebuild row %s: %w", language, message.ID, err) + } + if err := insertLexicalMessageTx(ctx, tx, table, message, content); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate %s rebuild rows: %w", table, err) + } + if err := optimizeFTS(ctx, tx, table); err != nil { + return err + } + return tx.Commit() +} + +func (s *Store) upsertLexicalMessageTx( + ctx context.Context, + tx *sql.Tx, + message MessageRecord, + tokenized map[string]string, +) error { + for _, language := range s.lexicalLanguages() { + table := lexicalFTSTable(language) + rowID, ok := messageFTSRowID(message.ID) + if !ok { + continue + } + if _, err := tx.ExecContext(ctx, "delete from "+table+" where rowid = ?", rowID); err != nil { + return err + } + if message.DeletedAt == "" { + if err := insertLexicalMessageTx(ctx, tx, table, message, tokenized[language]); err != nil { + return err + } + } + } + return nil +} + +func (s *Store) deleteLexicalMessagesTx(ctx context.Context, tx *sql.Tx, column string, value any) error { + if column != "rowid" && column != "guild_id" { + return fmt.Errorf("unsupported lexical delete column %q", column) + } + for _, language := range s.lexicalLanguages() { + if _, err := tx.ExecContext(ctx, "delete from "+lexicalFTSTable(language)+" where "+column+" = ?", value); err != nil { + return err + } + } + return nil +} + +func insertLexicalMessageTx(ctx context.Context, tx *sql.Tx, table string, message MessageRecord, content string) error { + rowID, ok := messageFTSRowID(message.ID) + if !ok { + return nil + } + _, err := tx.ExecContext(ctx, ` + insert into `+table+`( + rowid, message_id, guild_id, channel_id, author_id, author_name, channel_name, content + ) values(?, ?, ?, ?, ?, ?, ?, ?) + `, rowID, message.ID, message.GuildID, message.ChannelID, nullable(message.AuthorID), message.AuthorName, message.ChannelName, content) + return err +} + +func lexicalFTSTable(language string) string { + switch language { + case "ko", "ja", "zh", "ar": + return "message_fts_" + language + default: + panic("unsupported lexical language: " + language) + } +} + +func lexicalFTSScope(language string) string { + return "schema:" + lexicalFTSTable(language) + "_version" +} + +func isMessageFTSTable(table string) bool { + return table == "message_fts" || + table == "message_fts_ko" || + table == "message_fts_ja" || + table == "message_fts_zh" || + table == "message_fts_ar" +} + +func createLexicalFTSSQL(table string) string { + return `create virtual table ` + table + ` using fts5( + message_id unindexed, + guild_id unindexed, + channel_id unindexed, + author_id unindexed, + author_name, + channel_name, + content, + tokenize = 'unicode61 remove_diacritics 0' + )` +} + +func closeLexicalTokenizers(tokenizers map[string]LexicalTokenizer) { + for _, tokenizer := range tokenizers { + _ = tokenizer.Close() + } +} + +const lexicalRebuildRowsSQL = ` + select + m.id, + m.guild_id, + m.channel_id, + coalesce(m.author_id, ''), + coalesce( + json_extract(m.raw_json, '$.member.nick'), + json_extract(m.raw_json, '$.author.global_name'), + json_extract(m.raw_json, '$.author.username'), + '' + ), + coalesce(c.name, ''), + m.normalized_content + from messages m + left join channels c on c.id = m.channel_id + where m.deleted_at is null + order by cast(m.id as integer) +` diff --git a/internal/store/lexical_python.go b/internal/store/lexical_python.go new file mode 100644 index 00000000..f53135db --- /dev/null +++ b/internal/store/lexical_python.go @@ -0,0 +1,312 @@ +package store + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type OpenOptions struct { + LexicalLanguages []string + LexicalPython string +} + +type pythonLexicalTokenizer struct { + language string + command *exec.Cmd + stdin io.WriteCloser + stdout *bufio.Scanner + stderr lockedBuffer + mutex sync.Mutex +} + +type lockedBuffer struct { + mutex sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(data []byte) (int, error) { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.buffer.Write(data) +} + +func (b *lockedBuffer) String() string { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.buffer.String() +} + +type pythonLexicalResponse struct { + Ready bool `json:"ready,omitempty"` + Tokens string `json:"tokens,omitempty"` + Error string `json:"error,omitempty"` +} + +func OpenWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { + tokenizers, err := newPythonLexicalTokenizers(opts) + if err != nil { + return nil, err + } + return openWithLexicalTokenizers(ctx, path, tokenizers) +} + +func OpenReadOnlyWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { + tokenizers, err := newPythonLexicalTokenizers(opts) + if err != nil { + return nil, err + } + return openReadOnlyWithLexicalTokenizers(ctx, path, tokenizers) +} + +func newPythonLexicalTokenizers(opts OpenOptions) (map[string]LexicalTokenizer, error) { + if len(opts.LexicalLanguages) == 0 { + return nil, nil + } + python := strings.TrimSpace(opts.LexicalPython) + if python == "" { + python = "python3" + } + if strings.HasPrefix(python, "~/") { + home, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("resolve lexical Python home directory: %w", err) + } + python = filepath.Join(home, strings.TrimPrefix(python, "~/")) + } + tokenizers := make(map[string]LexicalTokenizer, len(opts.LexicalLanguages)) + for _, language := range opts.LexicalLanguages { + tokenizer, err := startPythonLexicalTokenizer(python, language) + if err != nil { + closeLexicalTokenizers(tokenizers) + return nil, err + } + tokenizers[language] = tokenizer + } + return tokenizers, nil +} + +func startPythonLexicalTokenizer( + python string, + language string, +) (*pythonLexicalTokenizer, error) { + command, err := newPythonLexicalCommand(python, language) + if err != nil { + return nil, err + } + return startPythonLexicalTokenizerCommand(command, language) +} + +func startPythonLexicalTokenizerCommand( + command *exec.Cmd, + language string, +) (*pythonLexicalTokenizer, error) { + tokenizer := &pythonLexicalTokenizer{language: language, command: command} + stdin, err := command.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := command.StdoutPipe() + if err != nil { + _ = stdin.Close() + return nil, err + } + command.Stderr = &tokenizer.stderr + tokenizer.stdin = stdin + tokenizer.stdout = bufio.NewScanner(stdout) + tokenizer.stdout.Buffer(make([]byte, 4096), 8*1024*1024) + if err := command.Start(); err != nil { + return nil, fmt.Errorf("start %s lexical tokenizer with %s: %w", language, command.Path, err) + } + response, err := tokenizer.readStartupResponse() + if err != nil { + _ = tokenizer.Close() + return nil, fmt.Errorf("initialize %s lexical tokenizer: %w", language, err) + } + if !response.Ready { + _ = tokenizer.Close() + return nil, fmt.Errorf("initialize %s lexical tokenizer: %s", language, response.Error) + } + return tokenizer, nil +} + +func (p *pythonLexicalTokenizer) readStartupResponse() (pythonLexicalResponse, error) { + type startupResult struct { + response pythonLexicalResponse + err error + } + result := make(chan startupResult, 1) + go func() { + response, err := p.readResponse() + result <- startupResult{response: response, err: err} + }() + select { + case startup := <-result: + return startup.response, startup.err + case <-time.After(30 * time.Second): + _ = p.command.Process.Kill() + return pythonLexicalResponse{}, errors.New("startup timed out after 30s") + } +} + +func (p *pythonLexicalTokenizer) Tokenize(ctx context.Context, text string) (string, error) { + p.mutex.Lock() + defer p.mutex.Unlock() + if err := ctx.Err(); err != nil { + return "", err + } + request, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return "", err + } + if _, err := p.stdin.Write(append(request, '\n')); err != nil { + return "", fmt.Errorf("write tokenizer request: %w", err) + } + response, err := p.readResponseContext(ctx) + if err != nil { + return "", err + } + if response.Error != "" { + return "", errors.New(response.Error) + } + return response.Tokens, nil +} + +func (p *pythonLexicalTokenizer) readResponseContext(ctx context.Context) (pythonLexicalResponse, error) { + readCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + type responseResult struct { + response pythonLexicalResponse + err error + } + result := make(chan responseResult, 1) + go func() { + response, err := p.readResponse() + result <- responseResult{response: response, err: err} + }() + select { + case response := <-result: + return response.response, response.err + case <-readCtx.Done(): + _ = p.command.Process.Kill() + return pythonLexicalResponse{}, fmt.Errorf("tokenizer response: %w", readCtx.Err()) + } +} + +func (p *pythonLexicalTokenizer) readResponse() (pythonLexicalResponse, error) { + if !p.stdout.Scan() { + err := p.stdout.Err() + if err == nil { + err = io.EOF + } + detail := strings.TrimSpace(p.stderr.String()) + if detail != "" { + return pythonLexicalResponse{}, fmt.Errorf("%w: %s", err, detail) + } + return pythonLexicalResponse{}, err + } + var response pythonLexicalResponse + if err := json.Unmarshal(p.stdout.Bytes(), &response); err != nil { + return pythonLexicalResponse{}, fmt.Errorf("decode tokenizer response: %w", err) + } + return response, nil +} + +func (p *pythonLexicalTokenizer) Close() error { + if p == nil || p.command == nil || p.command.Process == nil { + return nil + } + _ = p.stdin.Close() + err := p.command.Wait() + if errors.Is(err, os.ErrProcessDone) { + return nil + } + return err +} + +const pythonLexicalWorker = ` +import json +import re +import sys +import unicodedata + +language = sys.argv[1] + +try: + if language == "default": + engine = None + elif language == "ko": + from kiwipiepy import Kiwi + engine = Kiwi() + elif language == "ja": + from sudachipy import dictionary, tokenizer as sudachi_tokenizer + engine = dictionary.Dictionary().create() + split_mode = sudachi_tokenizer.Tokenizer.SplitMode.A + elif language == "zh": + import jieba + engine = jieba + elif language == "ar": + import snowballstemmer + engine = snowballstemmer.stemmer("arabic") + else: + raise RuntimeError("unsupported lexical language: " + language) +except Exception as error: + print(json.dumps({"ready": False, "error": str(error)}, ensure_ascii=False), flush=True) + raise SystemExit(2) + +print(json.dumps({"ready": True}, ensure_ascii=False), flush=True) + +def unique(tokens): + output = [] + seen = set() + for token in tokens: + token = unicodedata.normalize("NFKC", token).strip().lower() + if token and token not in seen: + seen.add(token) + output.append(token) + return output + +def tokenize(text): + if language == "default": + return unique(re.findall(r"[^\W\d_]+", text, flags=re.UNICODE)) + if language == "ko": + return unique(token.form for token in engine.tokenize(text) if not token.tag.startswith("S")) + if language == "ja": + output = [] + for token in engine.tokenize(text, split_mode): + output.append(token.surface()) + base = token.dictionary_form() + if base != "*": + output.append(base) + return unique(output) + if language == "zh": + return unique(engine.cut_for_search(text)) + words = re.findall(r"[^\W\d_]+", text, flags=re.UNICODE) + normalized = [re.sub(r"[\u064b-\u065f\u0670\u0640]", "", word) for word in words] + variants = list(normalized) + for word in normalized: + if word.startswith(("وال", "فال", "بال", "كال", "لال")) and len(word) > 4: + variants.append(word[3:]) + elif word.startswith("ال") and len(word) > 3: + variants.append(word[2:]) + elif word[:1] in ("و", "ف", "ب", "ك", "ل") and len(word) > 3: + variants.append(word[1:]) + return unique(variants + engine.stemWords(variants)) + +for line in sys.stdin: + try: + request = json.loads(line) + print(json.dumps({"tokens": " ".join(tokenize(request.get("text", "")))}, ensure_ascii=False), flush=True) + except Exception as error: + print(json.dumps({"error": str(error)}, ensure_ascii=False), flush=True) +` diff --git a/internal/store/lexical_python_command_unix.go b/internal/store/lexical_python_command_unix.go new file mode 100644 index 00000000..5dab4f33 --- /dev/null +++ b/internal/store/lexical_python_command_unix.go @@ -0,0 +1,24 @@ +//go:build !windows + +package store + +import ( + "os/exec" +) + +func newPythonLexicalCommand( + python string, + language string, +) (*exec.Cmd, error) { + return &exec.Cmd{ + Path: "/usr/bin/env", + Args: []string{ + "/usr/bin/env", + python, + "-u", + "-c", + pythonLexicalWorker, + language, + }, + }, nil +} diff --git a/internal/store/lexical_python_command_windows.go b/internal/store/lexical_python_command_windows.go new file mode 100644 index 00000000..304f0965 --- /dev/null +++ b/internal/store/lexical_python_command_windows.go @@ -0,0 +1,28 @@ +//go:build windows + +package store + +import ( + "fmt" + "os/exec" +) + +func newPythonLexicalCommand( + python string, + language string, +) (*exec.Cmd, error) { + args := []string{"-u", "-c", pythonLexicalWorker, language} + switch python { + case "python", "python.exe": + return exec.Command("python", args...), nil + case "python3", "python3.exe": + return exec.Command("python3", args...), nil + case "py", "py.exe": + return exec.Command("py", append([]string{"-3"}, args...)...), nil + default: + return nil, fmt.Errorf( + "unsupported Windows lexical Python launcher %q; use python, python3, or py", + python, + ) + } +} diff --git a/internal/store/lexical_python_test.go b/internal/store/lexical_python_test.go new file mode 100644 index 00000000..e4991912 --- /dev/null +++ b/internal/store/lexical_python_test.go @@ -0,0 +1,143 @@ +package store + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPythonLexicalTokenizerCommandProtocol(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizerCommand( + lexicalHelperCommand("ready"), + "test", + ) + require.NoError(t, err) + + tokens, err := tokenizer.Tokenize(context.Background(), "mixed text") + require.NoError(t, err) + require.Equal(t, "mixed text tokenized", tokens) + require.NoError(t, tokenizer.Close()) +} + +func TestPythonLexicalTokenizerCommandStartupError(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizerCommand( + lexicalHelperCommand("startup-error"), + "test", + ) + require.Nil(t, tokenizer) + require.ErrorContains(t, err, "missing tokenizer package") +} + +func TestPythonLexicalTokenizerCommandResponseError(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizerCommand( + lexicalHelperCommand("response-error"), + "test", + ) + require.NoError(t, err) + defer func() { _ = tokenizer.Close() }() + + _, err = tokenizer.Tokenize(context.Background(), "mixed text") + require.ErrorContains(t, err, "tokenization failed") +} + +func TestPythonLexicalTokenizerHonorsCanceledContext(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizerCommand( + lexicalHelperCommand("ready"), + "test", + ) + require.NoError(t, err) + defer func() { _ = tokenizer.Close() }() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = tokenizer.Tokenize(ctx, "mixed text") + require.ErrorIs(t, err, context.Canceled) +} + +func TestNewPythonLexicalTokenizersDisabled(t *testing.T) { + tokenizers, err := newPythonLexicalTokenizers(OpenOptions{}) + require.NoError(t, err) + require.Nil(t, tokenizers) +} + +func TestPythonLexicalTokenizerDefaultWorker(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizer("python3", "default") + require.NoError(t, err) + defer func() { _ = tokenizer.Close() }() + + tokens, err := tokenizer.Tokenize(context.Background(), "Mixed CASE 123") + require.NoError(t, err) + require.Equal(t, "mixed case", tokens) +} + +func TestNewPythonLexicalTokenizersExpandsHomePath(t *testing.T) { + python, err := exec.LookPath("python3") + require.NoError(t, err) + home := t.TempDir() + require.NoError(t, os.Symlink(python, filepath.Join(home, "python3"))) + t.Setenv("HOME", home) + + tokenizers, err := newPythonLexicalTokenizers(OpenOptions{ + LexicalLanguages: []string{"default"}, + LexicalPython: "~/python3", + }) + require.NoError(t, err) + require.Contains(t, tokenizers, "default") + closeLexicalTokenizers(tokenizers) +} + +func TestOpenWithOptionsReportsMissingPython(t *testing.T) { + _, err := OpenWithOptions(context.Background(), filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ + LexicalLanguages: []string{"ko"}, + LexicalPython: "/definitely/missing/discrawl-python", + }) + require.ErrorContains(t, err, "initialize ko lexical tokenizer") +} + +func lexicalHelperCommand(mode string) *exec.Cmd { + return &exec.Cmd{ + Path: os.Args[0], + Args: []string{os.Args[0], "-test.run=TestLexicalTokenizerHelperProcess", "--", mode}, + Env: append(os.Environ(), "DISCRAWL_LEXICAL_HELPER=1"), + } +} + +func TestLexicalTokenizerHelperProcess(t *testing.T) { + if os.Getenv("DISCRAWL_LEXICAL_HELPER") != "1" { + return + } + mode := os.Args[len(os.Args)-1] + if mode == "startup-error" { + fmt.Println(`{"error":"missing tokenizer package"}`) + os.Exit(2) + } + fmt.Println(`{"ready":true}`) + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + var request map[string]string + if err := json.Unmarshal(scanner.Bytes(), &request); err != nil { + fmt.Printf("{\"error\":%q}\n", err.Error()) + continue + } + if mode == "response-error" { + fmt.Println(`{"error":"tokenization failed"}`) + continue + } + response, err := json.Marshal(map[string]string{ + "tokens": request["text"] + " tokenized", + }) + if err != nil { + fmt.Printf("{\"error\":%q}\n", err.Error()) + continue + } + fmt.Println(string(response)) + } + os.Exit(0) +} diff --git a/internal/store/lexical_search.go b/internal/store/lexical_search.go new file mode 100644 index 00000000..fd4ffaed --- /dev/null +++ b/internal/store/lexical_search.go @@ -0,0 +1,150 @@ +package store + +import ( + "context" + "fmt" + "strings" + + "github.com/openclaw/crawlkit/vector" +) + +func (s *Store) searchMessagesMultilingual( + ctx context.Context, + opts SearchOptions, +) ([]SearchResult, error) { + if strings.TrimSpace(opts.Query) == "" { + return nil, nil + } + if opts.Limit <= 0 { + opts.Limit = 20 + } + candidateLimit := searchCandidateLimit(opts.Limit) + rankings := make([][]SearchResult, 0, len(s.lexicalTokenizers)+1) + + defaultResults, err := s.searchMessagesFTSTable( + ctx, + "message_fts", + normalizeFTSQuery(opts.Query), + opts, + candidateLimit, + ) + if err != nil { + if !shouldSearchFallback(err) { + return nil, err + } + return s.searchFallback(ctx, opts) + } + rankings = append(rankings, defaultResults) + + for _, language := range s.lexicalLanguages() { + query, err := s.lexicalTokenizers[language].Tokenize(ctx, opts.Query) + if err != nil { + return nil, fmt.Errorf("tokenize %s query: %w", language, err) + } + query = normalizeFTSQuery(query) + if query == "" { + continue + } + results, err := s.searchMessagesFTSTable( + ctx, + lexicalFTSTable(language), + query, + opts, + candidateLimit, + ) + if err != nil { + return nil, err + } + rankings = append(rankings, results) + } + return fuseLexicalSearchResults(rankings, opts.Limit), nil +} + +func (s *Store) searchMessagesFTSTable( + ctx context.Context, + table string, + queryText string, + opts SearchOptions, + limit int, +) ([]SearchResult, error) { + args := []any{queryText} + clauses := []string{table + " match ?"} + if len(opts.GuildIDs) > 0 { + clauses = append(clauses, table+".guild_id in ("+placeholders(len(opts.GuildIDs))+")") + for _, guildID := range opts.GuildIDs { + args = append(args, guildID) + } + } + if strings.TrimSpace(opts.Channel) != "" { + clauses = append(clauses, "("+table+".channel_id = ? or "+table+".channel_name like ?)") + args = append(args, opts.Channel, "%"+opts.Channel+"%") + } + if strings.TrimSpace(opts.Author) != "" { + clauses = append(clauses, "("+table+".author_id = ? or "+table+".author_name like ?)") + args = append(args, opts.Author, "%"+opts.Author+"%") + } + if !opts.IncludeEmpty { + clauses = append(clauses, "trim(coalesce(m.normalized_content, '')) <> ''") + } + args = append(args, limit) + queryCtx, cancel := withQueryTimeout(ctx) + defer cancel() + rows, err := s.db.QueryContext(queryCtx, ` + select `+table+`.message_id + from `+table+` + join messages m on m.id = `+table+`.message_id + where m.deleted_at is null + and `+strings.Join(clauses, " and ")+` + order by bm25(`+table+`) asc, `+table+`.rowid desc + limit ? + `, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + ids := make([]string, 0, limit) + for rows.Next() { + var messageID string + if err := rows.Scan(&messageID); err != nil { + return nil, err + } + ids = append(ids, messageID) + } + if err := rows.Err(); err != nil { + return nil, err + } + details, err := s.searchResultDetails(queryCtx, ids) + if err != nil { + return nil, err + } + results := make([]SearchResult, 0, len(ids)) + for _, messageID := range ids { + if result, ok := details[messageID]; ok { + results = append(results, result) + } + } + return results, nil +} + +func fuseLexicalSearchResults(rankings [][]SearchResult, limit int) []SearchResult { + if limit <= 0 { + limit = 20 + } + ids := make([]func(SearchResult) string, len(rankings)) + weights := make([]float64, len(rankings)) + for i := range rankings { + ids[i] = func(result SearchResult) string { + return result.MessageID + } + weights[i] = 1 + } + fused := vector.ReciprocalRankFusion(rankings, ids, weights, rrfK) + if len(fused) > limit { + fused = fused[:limit] + } + results := make([]SearchResult, 0, len(fused)) + for _, entry := range fused { + results = append(results, entry.Item) + } + return results +} diff --git a/internal/store/multilingual_benchmark_test.go b/internal/store/multilingual_benchmark_test.go new file mode 100644 index 00000000..60ac3ae0 --- /dev/null +++ b/internal/store/multilingual_benchmark_test.go @@ -0,0 +1,131 @@ +package store + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type multilingualBenchmarkCase struct { + language string + id string + content string + query string +} + +func TestMultilingualLexicalQualityBenchmark(t *testing.T) { + if os.Getenv("DISCRAWL_TOKENIZER_E2E") != "1" { + t.Skip("set DISCRAWL_TOKENIZER_E2E=1 with optional tokenizer packages installed") + } + ctx := context.Background() + root := t.TempDir() + baseline, err := Open(ctx, filepath.Join(root, "baseline.db")) + require.NoError(t, err) + defer func() { _ = baseline.Close() }() + multilingual, err := OpenWithOptions(ctx, filepath.Join(root, "multilingual.db"), OpenOptions{ + LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, + LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), + }) + require.NoError(t, err) + defer func() { _ = multilingual.Close() }() + + cases := multilingualBenchmarkCases() + base := time.Date(2026, 8, 18, 12, 0, 0, 0, time.UTC) + for i, item := range cases { + message := MessageRecord{ + ID: item.id, + GuildID: "g1", + ChannelID: "c1", + CreatedAt: base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339Nano), + Content: item.content, + NormalizedContent: item.content, + RawJSON: `{}`, + } + require.NoError(t, baseline.UpsertMessage(ctx, message)) + require.NoError(t, multilingual.UpsertMessage(ctx, message)) + } + baselineBytes := sqliteDatabaseBytes(t, baseline) + multilingualBytes := sqliteDatabaseBytes(t, multilingual) + t.Logf( + "database pages: unicode61=%d bytes multilingual=%d bytes (%.2fx)", + baselineBytes, + multilingualBytes, + float64(multilingualBytes)/float64(baselineBytes), + ) + + baselineHits := make(map[string]int) + multilingualHits := make(map[string]int) + totals := make(map[string]int) + for _, item := range cases { + totals[item.language]++ + baselineResults, err := baseline.SearchMessages(ctx, SearchOptions{Query: item.query, Limit: 5}) + require.NoError(t, err) + if containsSearchResult(baselineResults, item.id) { + baselineHits[item.language]++ + } + multilingualResults, err := multilingual.SearchMessages(ctx, SearchOptions{Query: item.query, Limit: 5}) + require.NoError(t, err) + if containsSearchResult(multilingualResults, item.id) { + multilingualHits[item.language]++ + } + } + for _, language := range []string{"ko", "ja", "zh", "ar"} { + t.Logf( + "%s recall@5: unicode61=%d/%d multilingual=%d/%d", + language, + baselineHits[language], + totals[language], + multilingualHits[language], + totals[language], + ) + require.Greater(t, multilingualHits[language], baselineHits[language], language) + require.Equal(t, totals[language], multilingualHits[language], language) + } +} + +func sqliteDatabaseBytes(t *testing.T, s *Store) int64 { + t.Helper() + var pageCount int64 + var pageSize int64 + require.NoError(t, s.DB().QueryRowContext(t.Context(), `pragma page_count`).Scan(&pageCount)) + require.NoError(t, s.DB().QueryRowContext(t.Context(), `pragma page_size`).Scan(&pageSize)) + return pageCount * pageSize +} + +func containsSearchResult(results []SearchResult, messageID string) bool { + for _, result := range results { + if result.MessageID == messageID { + return true + } + } + return false +} + +func multilingualBenchmarkCases() []multilingualBenchmarkCase { + return []multilingualBenchmarkCase{ + {language: "ko", id: "ko-1", content: "오늘저녁먹음", query: "저녁"}, + {language: "ko", id: "ko-2", content: "서울맛집추천", query: "맛집"}, + {language: "ko", id: "ko-3", content: "프로젝트검색기능", query: "검색"}, + {language: "ko", id: "ko-4", content: "회의기록정리", query: "기록"}, + {language: "ko", id: "ko-5", content: "운동계획세움", query: "계획"}, + {language: "ja", id: "ja-1", content: "東京都庁に行きます", query: "東京"}, + {language: "ja", id: "ja-2", content: "自然言語処理を学ぶ", query: "言語"}, + {language: "ja", id: "ja-3", content: "検索機能を改善する", query: "検索"}, + {language: "ja", id: "ja-4", content: "会議記録を整理する", query: "記録"}, + {language: "ja", id: "ja-5", content: "機械学習モデル", query: "学習"}, + {language: "zh", id: "zh-1", content: "自然语言处理很有趣", query: "语言"}, + {language: "zh", id: "zh-2", content: "北京大学校园很美", query: "大学"}, + {language: "zh", id: "zh-3", content: "搜索功能需要改进", query: "搜索"}, + {language: "zh", id: "zh-4", content: "会议记录已经完成", query: "记录"}, + {language: "zh", id: "zh-5", content: "机器学习模型上线", query: "学习"}, + {language: "ar", id: "ar-1", content: "والكتاب مفيد للطلاب", query: "كتاب"}, + {language: "ar", id: "ar-2", content: "والمدرسة تفتح صباحا", query: "مدرسة"}, + {language: "ar", id: "ar-3", content: "فالاجتماع مهم اليوم", query: "اجتماع"}, + {language: "ar", id: "ar-4", content: "بالسجل تفاصيل كاملة", query: "سجل"}, + {language: "ar", id: "ar-5", content: "كالبرنامج سريع جدا", query: "برنامج"}, + } +} diff --git a/internal/store/multilingual_edges_test.go b/internal/store/multilingual_edges_test.go new file mode 100644 index 00000000..bf271f3d --- /dev/null +++ b/internal/store/multilingual_edges_test.go @@ -0,0 +1,101 @@ +package store + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMultilingualSearchPreservesMetadataFilters(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: func(text string) string { return text }}, + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + now := time.Now().UTC() + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "wanted", GuildID: "g1", ChannelID: "c1", ChannelName: "alpha", + AuthorID: "u1", AuthorName: "alice", CreatedAt: now.Format(time.RFC3339Nano), + Content: "needle", NormalizedContent: "needle", RawJSON: `{}`, + })) + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "other", GuildID: "g2", ChannelID: "c2", ChannelName: "beta", + AuthorID: "u2", AuthorName: "bob", CreatedAt: now.Add(time.Minute).Format(time.RFC3339Nano), + Content: "needle", NormalizedContent: "needle", RawJSON: `{}`, + })) + + results, err := s.SearchMessages(ctx, SearchOptions{ + Query: "needle", GuildIDs: []string{"g1"}, Channel: "alpha", Author: "alice", + }) + require.NoError(t, err) + require.Equal(t, []string{"wanted"}, searchResultIDs(results)) + + results, err = s.SearchMessages(ctx, SearchOptions{ + Query: "needle", GuildIDs: []string{"g1"}, Channel: "missing", + }) + require.NoError(t, err) + require.Empty(t, results) +} + +func TestMultilingualSearchReportsQueryTokenizerFailure(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: func(text string) string { return text }}, + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + s.lexicalTokenizers["ko"] = failingLexicalTokenizer{err: errors.New("query tokenizer failed")} + + _, err = s.SearchMessages(ctx, SearchOptions{Query: "needle"}) + require.ErrorContains(t, err, "tokenize ko query") +} + +func TestMultilingualSearchFallsBackWhenDefaultFTSIsMissing(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: func(text string) string { return text }}, + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "fallback", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "needle", NormalizedContent: "needle", RawJSON: `{}`, + })) + _, err = s.DB().ExecContext(ctx, `drop table message_fts`) + require.NoError(t, err) + + results, err := s.SearchMessages(ctx, SearchOptions{Query: "needle", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"fallback"}, searchResultIDs(results)) +} + +func TestMultilingualDeleteRejectsUnsafeColumn(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), nil) + require.NoError(t, err) + defer func() { _ = s.Close() }() + tx, err := s.DB().BeginTx(ctx, nil) + require.NoError(t, err) + defer rollback(tx) + + err = s.deleteLexicalMessagesTx(ctx, tx, "message_id", "unsafe") + require.ErrorContains(t, err, "unsupported lexical delete column") +} + +func TestFuseLexicalSearchResultsUsesDefaultLimit(t *testing.T) { + ranking := make([]SearchResult, 25) + for i := range ranking { + ranking[i].MessageID = fmt.Sprintf("message-%02d", i) + } + results := fuseLexicalSearchResults([][]SearchResult{ranking}, 0) + require.Len(t, results, 20) + require.Equal(t, "message-00", results[0].MessageID) +} diff --git a/internal/store/multilingual_lifecycle_test.go b/internal/store/multilingual_lifecycle_test.go new file mode 100644 index 00000000..1cf30575 --- /dev/null +++ b/internal/store/multilingual_lifecycle_test.go @@ -0,0 +1,145 @@ +package store + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type failingLexicalTokenizer struct { + err error +} + +func (f failingLexicalTokenizer) Tokenize(context.Context, string) (string, error) { + return "", f.err +} + +func (failingLexicalTokenizer) Close() error { + return nil +} + +func TestMultilingualIndexesTrackBatchDeletesAndGuildPurge(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "저녁먹음": "저녁 먹 음", + "회의기록": "회의 기록", + })}, + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + now := time.Now().UTC() + require.NoError(t, s.UpsertMessages(ctx, []MessageMutation{ + {Record: MessageRecord{ + ID: "first", GuildID: "g1", ChannelID: "c1", + CreatedAt: now.Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + }}, + {Record: MessageRecord{ + ID: "second", GuildID: "g2", ChannelID: "c2", + CreatedAt: now.Add(time.Minute).Format(time.RFC3339Nano), + Content: "회의기록", NormalizedContent: "회의기록", RawJSON: `{}`, + }}, + })) + + results, err := s.SearchMessages(ctx, SearchOptions{Query: "저녁", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"first"}, searchResultIDs(results)) + require.NoError(t, s.MarkMessageDeleted( + ctx, + "g1", + "c1", + "first", + map[string]string{"deleted_at": now.Add(time.Hour).Format(time.RFC3339Nano)}, + )) + results, err = s.SearchMessages(ctx, SearchOptions{Query: "저녁", Limit: 10}) + require.NoError(t, err) + require.Empty(t, results) + + results, err = s.SearchMessages(ctx, SearchOptions{Query: "기록", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"second"}, searchResultIDs(results)) + require.NoError(t, s.DeleteGuildData(ctx, "g2")) + results, err = s.SearchMessages(ctx, SearchOptions{Query: "기록", Limit: 10}) + require.NoError(t, err) + require.Empty(t, results) +} + +func TestMultilingualIndexVersionSurvivesReopen(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "discrawl.db") + tokenizers := func() map[string]LexicalTokenizer { + return map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "저녁먹음": "저녁 먹 음", + })}, + } + } + s, err := openWithLexicalTokenizers(ctx, path, tokenizers()) + require.NoError(t, err) + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "ko", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + })) + require.NoError(t, s.Close()) + + s, err = openWithLexicalTokenizers(ctx, path, tokenizers()) + require.NoError(t, err) + defer func() { _ = s.Close() }() + results, err := s.SearchMessages(ctx, SearchOptions{Query: "저녁", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"ko"}, searchResultIDs(results)) +} + +func TestMultilingualIndexesSearchThroughReadOnlyStore(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "discrawl.db") + tokenizers := func() map[string]LexicalTokenizer { + return map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "저녁먹음": "저녁 먹 음", + })}, + } + } + writer, err := openWithLexicalTokenizers(ctx, path, tokenizers()) + require.NoError(t, err) + require.NoError(t, writer.UpsertMessage(ctx, MessageRecord{ + ID: "ko", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + })) + require.NoError(t, writer.Close()) + + reader, err := openReadOnlyWithLexicalTokenizers(ctx, path, tokenizers()) + require.NoError(t, err) + defer func() { _ = reader.Close() }() + results, err := reader.SearchMessages(ctx, SearchOptions{Query: "저녁", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"ko"}, searchResultIDs(results)) +} + +func TestMultilingualTokenizerFailureAbortsWrite(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), nil) + require.NoError(t, err) + defer func() { _ = s.Close() }() + s.lexicalTokenizers = map[string]LexicalTokenizer{ + "ko": failingLexicalTokenizer{err: errors.New("tokenizer unavailable")}, + } + + err = s.UpsertMessage(ctx, MessageRecord{ + ID: "ko", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + }) + require.ErrorContains(t, err, "tokenize ko text") + var count int + require.NoError(t, s.DB().QueryRowContext(ctx, `select count(*) from messages`).Scan(&count)) + require.Zero(t, count) +} diff --git a/internal/store/multilingual_python_e2e_test.go b/internal/store/multilingual_python_e2e_test.go new file mode 100644 index 00000000..0589ae88 --- /dev/null +++ b/internal/store/multilingual_python_e2e_test.go @@ -0,0 +1,52 @@ +package store + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPythonMultilingualLexicalSearchE2E(t *testing.T) { + if os.Getenv("DISCRAWL_TOKENIZER_E2E") != "1" { + t.Skip("set DISCRAWL_TOKENIZER_E2E=1 with optional tokenizer packages installed") + } + ctx := context.Background() + s, err := OpenWithOptions(ctx, filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ + LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, + LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + base := time.Date(2026, 8, 18, 12, 0, 0, 0, time.UTC) + fixtures := []struct { + id string + content string + query string + }{ + {id: "ko", content: "오늘 저녁먹음 기록", query: "저녁"}, + {id: "ja", content: "東京都庁に行きます", query: "東京"}, + {id: "zh", content: "自然语言处理很有趣", query: "语言"}, + {id: "ar", content: "والكتاب مفيد للطلاب", query: "كتاب"}, + } + for i, fixture := range fixtures { + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: fixture.id, + GuildID: "g1", + ChannelID: "c1", + CreatedAt: base.Add(time.Duration(i) * time.Minute).Format(time.RFC3339Nano), + Content: fixture.content, + NormalizedContent: fixture.content, + RawJSON: `{}`, + })) + } + for _, fixture := range fixtures { + results, err := s.SearchMessages(ctx, SearchOptions{Query: fixture.query, Limit: 10}) + require.NoError(t, err, fixture.id) + require.Contains(t, searchResultIDs(results), fixture.id, fixture.query) + } +} diff --git a/internal/store/multilingual_search_test.go b/internal/store/multilingual_search_test.go new file mode 100644 index 00000000..1b96ddc1 --- /dev/null +++ b/internal/store/multilingual_search_test.go @@ -0,0 +1,148 @@ +package store + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +type stubLexicalTokenizer struct { + tokenize func(string) string +} + +func (s stubLexicalTokenizer) Tokenize(_ context.Context, text string) (string, error) { + return s.tokenize(text), nil +} + +func (stubLexicalTokenizer) Close() error { + return nil +} + +func TestSearchMessagesMultilingualIndexesEachConfiguredAnalyzer(t *testing.T) { + ctx := context.Background() + tokenizers := map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "저녁먹음": "저녁 먹 음", + })}, + "ja": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "東京都庁": "東京 都庁", + })}, + "zh": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "自然语言处理": "自然 语言 处理", + })}, + "ar": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "والكتاب": "و ال كتاب", + })}, + } + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), tokenizers) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + base := time.Date(2026, 8, 18, 12, 0, 0, 0, time.UTC) + messages := []MessageRecord{ + {ID: "ko", GuildID: "g1", ChannelID: "c1", CreatedAt: base.Format(time.RFC3339Nano), Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`}, + {ID: "ja", GuildID: "g1", ChannelID: "c1", CreatedAt: base.Add(time.Minute).Format(time.RFC3339Nano), Content: "東京都庁", NormalizedContent: "東京都庁", RawJSON: `{}`}, + {ID: "zh", GuildID: "g1", ChannelID: "c1", CreatedAt: base.Add(2 * time.Minute).Format(time.RFC3339Nano), Content: "自然语言处理", NormalizedContent: "自然语言处理", RawJSON: `{}`}, + {ID: "ar", GuildID: "g1", ChannelID: "c1", CreatedAt: base.Add(3 * time.Minute).Format(time.RFC3339Nano), Content: "والكتاب", NormalizedContent: "والكتاب", RawJSON: `{}`}, + } + for _, message := range messages { + require.NoError(t, s.UpsertMessage(ctx, message)) + } + + for query, wantID := range map[string]string{ + "저녁": "ko", + "東京": "ja", + "语言": "zh", + "كتاب": "ar", + } { + results, err := s.SearchMessages(ctx, SearchOptions{Query: query, Limit: 10}) + require.NoError(t, err, query) + require.Equal(t, []string{wantID}, searchResultIDs(results), query) + } +} + +func TestSearchMessagesMultilingualRRFCombinesAndDeduplicates(t *testing.T) { + ctx := context.Background() + tokenizers := map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: func(text string) string { + text = strings.ReplaceAll(text, "검색", "shared") + return strings.ReplaceAll(text, "検索", "") + }}, + "ja": stubLexicalTokenizer{tokenize: func(text string) string { + text = strings.ReplaceAll(text, "検索", "shared") + return strings.ReplaceAll(text, "검색", "") + }}, + } + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), tokenizers) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + base := time.Date(2026, 8, 18, 12, 0, 0, 0, time.UTC) + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "mixed", GuildID: "g1", ChannelID: "c1", CreatedAt: base.Format(time.RFC3339Nano), + Content: "검색 検索", NormalizedContent: "검색 検索", RawJSON: `{}`, + })) + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "single", GuildID: "g1", ChannelID: "c1", CreatedAt: base.Add(time.Minute).Format(time.RFC3339Nano), + Content: "검색 only", NormalizedContent: "검색 only", RawJSON: `{}`, + })) + + results, err := s.SearchMessages(ctx, SearchOptions{Query: "검색 検索", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"mixed", "single"}, searchResultIDs(results)) +} + +func TestRebuildSearchIndexesRebuildsMultilingualTables(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{"저녁먹음": "저녁 먹 음"})}, + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "ko", GuildID: "g1", ChannelID: "c1", CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + })) + _, err = s.DB().ExecContext(ctx, `delete from message_fts_ko`) + require.NoError(t, err) + require.NoError(t, s.RebuildSearchIndexes(ctx)) + + results, err := s.SearchMessages(ctx, SearchOptions{Query: "저녁", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"ko"}, searchResultIDs(results)) +} + +func TestSearchMessagesMultilingualHonorsIncludeEmpty(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: func(text string) string { return text }}, + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "empty", GuildID: "g1", ChannelID: "c1", AuthorName: "needle", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), RawJSON: `{}`, + })) + results, err := s.SearchMessages(ctx, SearchOptions{Query: "needle", Limit: 10}) + require.NoError(t, err) + require.Empty(t, results) + + results, err = s.SearchMessages(ctx, SearchOptions{Query: "needle", Limit: 10, IncludeEmpty: true}) + require.NoError(t, err) + require.Equal(t, []string{"empty"}, searchResultIDs(results)) +} + +func replaceLexicalTerms(replacements map[string]string) func(string) string { + return func(text string) string { + for from, to := range replacements { + text = strings.ReplaceAll(text, from, to) + } + return text + } +} diff --git a/internal/store/query.go b/internal/store/query.go index 94013dea..84fd0eda 100644 --- a/internal/store/query.go +++ b/internal/store/query.go @@ -105,6 +105,9 @@ func (s *Store) SearchMessages(ctx context.Context, opts SearchOptions) ([]Searc if opts.Limit <= 0 { opts.Limit = 20 } + if len(s.lexicalTokenizers) > 0 { + return s.searchMessagesMultilingual(ctx, opts) + } args := []any{normalizeFTSQuery(opts.Query)} clauses := []string{"message_fts match ?"} if len(opts.GuildIDs) > 0 { diff --git a/internal/store/store.go b/internal/store/store.go index a12586cf..903b0d86 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -23,9 +23,11 @@ const ( var ErrSchemaVersionMismatch = errors.New("database schema version mismatch") type Store struct { - db *sql.DB - q *storedb.Queries - path string + db *sql.DB + q *storedb.Queries + path string + baseClose func() error + lexicalTokenizers map[string]LexicalTokenizer } type CatalogIntegrity struct { @@ -132,31 +134,48 @@ type ChannelRow struct { } func Open(ctx context.Context, path string) (*Store, error) { + return openWithLexicalTokenizers(ctx, path, nil) +} + +func openBaseStore(ctx context.Context, path string) (*crawlstore.Store, error) { base, err := crawlstore.Open(ctx, crawlstore.Options{Path: path}) if err != nil { return nil, err } - db := base.DB() - store := &Store{db: db, q: storedb.New(db), path: path} - if err := store.migrate(ctx); err != nil { - _ = base.Close() - return nil, err - } - return store, nil + return base, nil +} + +func newStoreQueries(db *sql.DB) *storedb.Queries { + return storedb.New(db) } func OpenReadOnly(ctx context.Context, path string) (*Store, error) { + return openReadOnlyWithLexicalTokenizers(ctx, path, nil) +} + +func openReadOnlyWithLexicalTokenizers( + ctx context.Context, + path string, + tokenizers map[string]LexicalTokenizer, +) (*Store, error) { base, err := crawlstore.OpenReadOnly(ctx, path) if err != nil { + closeLexicalTokenizers(tokenizers) return nil, err } db := base.DB() - store := &Store{db: db, q: storedb.New(db), path: path} + store := &Store{ + db: db, + q: storedb.New(db), + path: path, + baseClose: base.Close, + lexicalTokenizers: tokenizers, + } if version, err := store.schemaVersion(ctx); err != nil { - _ = base.Close() + _ = store.Close() return nil, err } else if version != storeSchemaVersion { - _ = base.Close() + _ = store.Close() return nil, fmt.Errorf("%w: got %d want %d", ErrSchemaVersionMismatch, version, storeSchemaVersion) } return store, nil @@ -166,6 +185,10 @@ func (s *Store) Close() error { if s == nil || s.db == nil { return nil } + closeLexicalTokenizers(s.lexicalTokenizers) + if s.baseClose != nil { + return s.baseClose() + } return s.db.Close() } @@ -258,6 +281,9 @@ func (s *Store) RebuildSearchIndexes(ctx context.Context) error { if err := s.rebuildFTS(ctx); err != nil { return err } + if err := s.rebuildLexicalIndexes(ctx); err != nil { + return err + } if err := s.rebuildMemberFTS(ctx); err != nil { return err } @@ -268,6 +294,9 @@ func (s *Store) RebuildMessageSearchIndex(ctx context.Context) error { if err := s.rebuildFTS(ctx); err != nil { return err } + if err := s.rebuildLexicalIndexes(ctx); err != nil { + return err + } return s.stampSearchIndexVersions(ctx, true, false) } @@ -845,7 +874,7 @@ func (s *Store) rebuildFTS(ctx context.Context) error { } func configureFTSBulkLoad(ctx context.Context, tx *sql.Tx, table string) error { - if table != "message_fts" && table != "member_fts" { + if !isMessageFTSTable(table) && table != "member_fts" { return fmt.Errorf("unsupported fts table %q", table) } stmts := []string{ @@ -862,7 +891,7 @@ func configureFTSBulkLoad(ctx context.Context, tx *sql.Tx, table string) error { } func optimizeFTS(ctx context.Context, tx *sql.Tx, table string) error { - if table != "message_fts" && table != "member_fts" { + if !isMessageFTSTable(table) && table != "member_fts" { return fmt.Errorf("unsupported fts table %q", table) } if _, err := tx.ExecContext(ctx, fmt.Sprintf("insert into %s(%s) values('optimize')", table, table)); err != nil { diff --git a/internal/store/write.go b/internal/store/write.go index 70174fec..f5de542b 100644 --- a/internal/store/write.go +++ b/internal/store/write.go @@ -227,6 +227,9 @@ func (s *Store) DeleteGuildData(ctx context.Context, guildID string) error { if _, err := tx.ExecContext(ctx, `delete from message_fts where guild_id = ?`, guildID); err != nil { return err } + if err := s.deleteLexicalMessagesTx(ctx, tx, "guild_id", guildID); err != nil { + return err + } if err := qtx.DeleteMessageEventsByGuild(ctx, guildID); err != nil { return err } @@ -263,12 +266,16 @@ func (s *Store) UpsertMessage(ctx context.Context, message MessageRecord) error } func (s *Store) UpsertMessageWithOptions(ctx context.Context, message MessageRecord, opts WriteOptions) error { + tokenized, err := s.tokenizeLexical(ctx, message.NormalizedContent) + if err != nil { + return err + } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer rollback(tx) - if err := upsertMessageTx(ctx, tx, s.q.WithTx(tx), message, opts); err != nil { + if err := s.upsertMessageTx(ctx, tx, s.q.WithTx(tx), message, opts, tokenized); err != nil { return err } return tx.Commit() @@ -278,17 +285,25 @@ func (s *Store) UpsertMessages(ctx context.Context, messages []MessageMutation) if len(messages) == 0 { return nil } + tokenized := make([]map[string]string, len(messages)) + for i, message := range messages { + var err error + tokenized[i], err = s.tokenizeLexical(ctx, message.Record.NormalizedContent) + if err != nil { + return err + } + } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer rollback(tx) qtx := s.q.WithTx(tx) - for _, message := range messages { + for i, message := range messages { if err := ctx.Err(); err != nil { return err } - if err := upsertMessageTx(ctx, tx, qtx, message.Record, message.Options); err != nil { + if err := s.upsertMessageTx(ctx, tx, qtx, message.Record, message.Options, tokenized[i]); err != nil { return err } if err := replaceAttachmentsTx(ctx, qtx, message.Record.ID, message.Attachments); err != nil { @@ -314,7 +329,14 @@ func (s *Store) UpsertMessages(ctx context.Context, messages []MessageMutation) return tx.Commit() } -func upsertMessageTx(ctx context.Context, tx *sql.Tx, qtx *storedb.Queries, message MessageRecord, opts WriteOptions) error { +func (s *Store) upsertMessageTx( + ctx context.Context, + tx *sql.Tx, + qtx *storedb.Queries, + message MessageRecord, + opts WriteOptions, + tokenized map[string]string, +) error { now := time.Now().UTC().Format(timeLayout) var previousNormalized sql.NullString previousErr := sql.ErrNoRows @@ -365,6 +387,9 @@ func upsertMessageTx(ctx context.Context, tx *sql.Tx, qtx *storedb.Queries, mess `, rowID, message.ID, message.GuildID, message.ChannelID, nullable(message.AuthorID), message.AuthorName, message.ChannelName, message.NormalizedContent); err != nil { return err } + if err := s.upsertLexicalMessageTx(ctx, tx, message, tokenized); err != nil { + return err + } } queueEmbedding := opts.EnqueueEmbedding && (errors.Is(previousErr, sql.ErrNoRows) || previousNormalized.String != message.NormalizedContent || !jobExists) if queueEmbedding { @@ -450,6 +475,9 @@ func (s *Store) markMessageDeleted( if _, err := tx.ExecContext(ctx, deleteMessageFTSByRowIDSQL, rowID); err != nil { return err } + if err := s.deleteLexicalMessagesTx(ctx, tx, "rowid", rowID); err != nil { + return err + } } if err := qtx.DeleteMessageEmbeddingsByMessage(ctx, messageID); err != nil { return err From e581712678e59b837c71acc8f238404fdccd554d Mon Sep 17 00:00:00 2001 From: "Jeffrey (Dongkyu) Kim" Date: Tue, 18 Aug 2026 18:56:11 +0900 Subject: [PATCH 2/5] fix(search): address multilingual review findings --- README.md | 1 + docs/README.md | 2 +- docs/benchmarks/multilingual-lexical.md | 23 ++ docs/commands/lexical.md | 26 +++ docs/configuration.md | 3 +- docs/guides/search-modes.md | 14 +- internal/cli/cli.go | 51 ++-- internal/cli/lexical_commands.go | 40 ++++ internal/cli/lexical_commands_test.go | 115 +++++++++ internal/cli/output.go | 6 + internal/store/lexical.go | 64 +++++ internal/store/lexical_install.go | 131 +++++++++++ internal/store/lexical_lazy.go | 34 +++ internal/store/lexical_lazy_test.go | 50 ++++ internal/store/lexical_python.go | 37 +-- internal/store/lexical_python_command.go | 102 ++++++++ internal/store/lexical_python_command_unix.go | 24 -- .../store/lexical_python_command_windows.go | 28 --- internal/store/lexical_python_test.go | 62 ++++- internal/store/lexical_review_test.go | 218 ++++++++++++++++++ internal/store/multilingual_lifecycle_test.go | 65 ++++++ 21 files changed, 998 insertions(+), 98 deletions(-) create mode 100644 docs/commands/lexical.md create mode 100644 internal/cli/lexical_commands.go create mode 100644 internal/cli/lexical_commands_test.go create mode 100644 internal/store/lexical_install.go create mode 100644 internal/store/lexical_lazy.go create mode 100644 internal/store/lexical_lazy_test.go create mode 100644 internal/store/lexical_python_command.go delete mode 100644 internal/store/lexical_python_command_unix.go delete mode 100644 internal/store/lexical_python_command_windows.go create mode 100644 internal/store/lexical_review_test.go diff --git a/README.md b/README.md index 756f8f33..8319ea2b 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ The full documentation lives at **[discrawl.sh](https://discrawl.sh/)**: - [Command reference](docs/README.md) - [Sync sources](docs/guides/sync-sources.md) - [Search modes](docs/guides/search-modes.md) +- [Optional lexical tokenizer installation](docs/commands/lexical.md) - [Multilingual lexical benchmark](docs/benchmarks/multilingual-lexical.md) - [Git snapshot workflows](docs/guides/git-snapshots.md) - [Configuration](docs/configuration.md) diff --git a/docs/README.md b/docs/README.md index b443cd04..98ee5941 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,7 +15,7 @@ Mirror Discord guilds into local SQLite. Search server history without depending ## Pick your path - **New here?** Read [Install](install.html) and run `discrawl init`. -- **Already have a bot?** Jump to [`sync`](commands/sync.html) and [`search`](commands/search.html). +- **Already have a bot?** Jump to [`sync`](commands/sync.html), [`search`](commands/search.html), and optional [`lexical install`](commands/lexical.html). - **Just want to read a shared archive?** Use [`subscribe`](commands/subscribe.html) for Git snapshots, or [`subscribe-cloud`](commands/subscribe-cloud.html) for a Worker-fronted archive - no Discord token needed. - **Need DM search?** [`wiretap`](commands/wiretap.html) imports local Discord Desktop cache. - **Want semantic search?** Configure [Embeddings](guides/embeddings.html), then run [`embed`](commands/embed.html). diff --git a/docs/benchmarks/multilingual-lexical.md b/docs/benchmarks/multilingual-lexical.md index d6aac50a..82ce5a89 100644 --- a/docs/benchmarks/multilingual-lexical.md +++ b/docs/benchmarks/multilingual-lexical.md @@ -46,3 +46,26 @@ duplicated while postings scale with enabled analyzers. The benchmark therefore supports a narrow claim: configured language fields substantially improve recall for these segmentation cases. It does not claim a universal 100-point gain on natural Discord query distributions. + +## Live verification transcript + +Captured from the command above: + +```text +=== RUN TestMultilingualLexicalQualityBenchmark +database pages: unicode61=233472 bytes multilingual=319488 bytes (1.37x) +ko recall@5: unicode61=0/5 multilingual=5/5 +ja recall@5: unicode61=0/5 multilingual=5/5 +zh recall@5: unicode61=0/5 multilingual=5/5 +ar recall@5: unicode61=0/5 multilingual=5/5 +--- PASS: TestMultilingualLexicalQualityBenchmark +=== RUN TestPythonMultilingualLexicalSearchE2E +--- PASS: TestPythonMultilingualLexicalSearchE2E +PASS +``` + +The selected-install path was separately exercised from an empty virtual +environment with `languages = ["ko"]`. `discrawl lexical install` installed +`kiwipiepy==0.23.2` plus its runtime dependencies, did not install Sudachi, +Jieba, or Snowball, and a subsequent CLI search for `저녁` returned the fixture +`오늘 저녁먹음 기록`. diff --git a/docs/commands/lexical.md b/docs/commands/lexical.md new file mode 100644 index 00000000..a077afc0 --- /dev/null +++ b/docs/commands/lexical.md @@ -0,0 +1,26 @@ +# `lexical` + +Installs only the optional tokenizer packages selected by +`search.lexical.languages`. + +## Usage + +```bash +discrawl lexical install +``` + +The configured `search.lexical.python` interpreter must belong to a virtual +environment. Discrawl refuses to install packages into a system Python. + +Package versions are pinned by Discrawl: + +| Language | Packages | +| --- | --- | +| `ko` | `kiwipiepy==0.23.2` | +| `ja` | `sudachipy==0.6.11`, `sudachidict_core==20260723` | +| `zh` | `jieba==0.42.1` | +| `ar` | `snowballstemmer==3.1.1` | + +Installation is always explicit. Opening an archive or running a search never +downloads or installs code. Tokenizer workers start lazily only when an enabled +language is first used for indexing or search. diff --git a/docs/configuration.md b/docs/configuration.md index 85f638ef..e23ea12b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -145,7 +145,8 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and - `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement` - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time - `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi), Japanese (`ja`, Sudachi), Chinese (`zh`, Jieba), and Arabic (`ar`, Snowball plus proclitic splitting). -- Multilingual lexical search requires the configured Python interpreter to provide `kiwipiepy`, `sudachipy`, `sudachidict_core`, `jieba`, and `snowballstemmer` for the enabled languages. Discrawl reports the missing module while opening the archive instead of silently falling back. +- `discrawl lexical install` installs only the pinned packages required by the configured languages and refuses system-Python installation. Search and sync never install packages implicitly. +- Tokenizer workers load lazily on first indexing or search use. Commands that only inspect metadata do not start Python or require tokenizer modules. - Each enabled language adds an independent FTS5 table. Index and query text pass through the same tokenizer, and results from the default plus language-specific tables are merged with reciprocal rank fusion. - After adding or changing `search.lexical.languages`, run a writer command such as `discrawl sync` once so the configured lexical tables are built. Read-only commands never mutate the archive; new and edited messages update the tables automatically during later syncs. - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search diff --git a/docs/guides/search-modes.md b/docs/guides/search-modes.md index e70f0813..9ffd323e 100644 --- a/docs/guides/search-modes.md +++ b/docs/guides/search-modes.md @@ -19,12 +19,10 @@ ### Optional multilingual lexical fields -Create an isolated Python environment and install only the tokenizer presets you enable: +Create an isolated Python environment: ```bash python3 -m venv ~/.local/share/discrawl/tokenizers -~/.local/share/discrawl/tokenizers/bin/python -m pip install \ - kiwipiepy sudachipy sudachidict_core jieba snowballstemmer ``` Then configure the fields: @@ -35,12 +33,22 @@ languages = ["ko", "ja", "zh", "ar"] python = "~/.local/share/discrawl/tokenizers/bin/python" # ~ is expanded ``` +Install only the packages selected by `languages`: + +```bash +discrawl lexical install +``` + Every message is analyzed into each configured field. This deliberately avoids language detection, so mixed-language Discord messages remain searchable through every enabled analyzer. Disk usage and indexing work increase with the number of fields; query-time RRF deduplicates message ids without mixing the different BM25 term statistics into one field. +Discrawl never installs packages during archive open, sync, or search. Workers +are loaded lazily on first use, while `lexical install` is an explicit, +virtual-environment-only network operation. + See [Multilingual lexical benchmark](../benchmarks/multilingual-lexical.html) for the reproducible targeted quality check and its storage tradeoff. diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 68c7680d..e9dcd8fb 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -128,6 +128,7 @@ var discrawlCommandSpecs = []discrawlCommandSpec{ {name: "cache-import", description: "Import Discord Desktop cache data (wiretap alias)."}, {name: "wiretap", description: "Import Discord Desktop cache data."}, {name: "search", description: "Search archived messages."}, + {name: "lexical", description: "Install configured multilingual lexical tokenizers."}, {name: "tui", description: "Explore the archive in an interactive terminal UI."}, {name: "messages", description: "List archived messages."}, {name: "digest", description: "Summarize recent archive activity."}, @@ -248,28 +249,29 @@ func parseKongArgs(target any, args []string, name string, stdout, stderr io.Wri } type runtime struct { - ctx context.Context - configPath string - cfg config.Config - stdout io.Writer - stderr io.Writer - json bool - plain bool - logger *slog.Logger - store *store.Store - client discordClient - syncer syncService - dbLockHeld bool - lockStarted time.Time - lockOperation string - lockToken string - lockTokenFree func() error - openStore func(context.Context, string) (*store.Store, error) - newDiscord func(config.Config) (discordClient, error) - newRemote func(config.Config) (remoteArchiveClient, error) - newSyncer func(syncer.Client, *store.Store, *slog.Logger) syncService - newEmbed func(config.EmbeddingsConfig) (embed.Provider, error) - now func() time.Time + ctx context.Context + configPath string + cfg config.Config + stdout io.Writer + stderr io.Writer + json bool + plain bool + logger *slog.Logger + store *store.Store + client discordClient + syncer syncService + dbLockHeld bool + lockStarted time.Time + lockOperation string + lockToken string + lockTokenFree func() error + openStore func(context.Context, string) (*store.Store, error) + newDiscord func(config.Config) (discordClient, error) + newRemote func(config.Config) (remoteArchiveClient, error) + newSyncer func(syncer.Client, *store.Store, *slog.Logger) syncService + newEmbed func(config.EmbeddingsConfig) (embed.Provider, error) + installLexical func(context.Context, string, []string) (store.LexicalInstallResult, error) + now func() time.Time } func crawlkitEmbeddingConfig(cfg config.EmbeddingsConfig) embed.Config { @@ -358,6 +360,11 @@ func (r *runtime) dispatch(rest []string) error { } autoShareUpdate := !hasBoolFlag(rest[1:], "--dm") return r.withLocalStoreRead(autoShareUpdate, func() error { return r.runSearch(rest[1:]) }) + case "lexical": + if hasHelpFlag(rest[1:]) { + return printCommandUsage(r.stdout, []string{"lexical"}) + } + return r.withConfig(func() error { return r.runLexical(rest[1:]) }) case "tui": if hasHelpArg(rest[1:]) { return r.runTUI(rest[1:]) diff --git a/internal/cli/lexical_commands.go b/internal/cli/lexical_commands.go new file mode 100644 index 00000000..794be976 --- /dev/null +++ b/internal/cli/lexical_commands.go @@ -0,0 +1,40 @@ +package cli + +import ( + "errors" + "fmt" + + "github.com/openclaw/discrawl/internal/store" +) + +type lexicalInstallOutput struct { + Languages []string `json:"languages"` + Packages []string `json:"packages"` + Python string `json:"python"` +} + +func (r *runtime) runLexical(args []string) error { + if len(args) != 1 || args[0] != "install" { + return usageErr(errors.New("usage: discrawl lexical install")) + } + if len(r.cfg.Search.Lexical.Languages) == 0 { + return configErr(errors.New("search.lexical.languages is empty")) + } + install := r.installLexical + if install == nil { + install = store.InstallLexicalPackages + } + result, err := install( + r.ctx, + r.cfg.Search.Lexical.Python, + r.cfg.Search.Lexical.Languages, + ) + if err != nil { + return configErr(fmt.Errorf("install lexical tokenizers: %w", err)) + } + return r.print(lexicalInstallOutput{ + Languages: append([]string(nil), r.cfg.Search.Lexical.Languages...), + Packages: result.Packages, + Python: r.cfg.Search.Lexical.Python, + }) +} diff --git a/internal/cli/lexical_commands_test.go b/internal/cli/lexical_commands_test.go new file mode 100644 index 00000000..02e82c25 --- /dev/null +++ b/internal/cli/lexical_commands_test.go @@ -0,0 +1,115 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/openclaw/discrawl/internal/config" + "github.com/openclaw/discrawl/internal/store" + "github.com/stretchr/testify/require" +) + +func TestRunLexicalInstallUsesConfiguredLanguages(t *testing.T) { + var stdout bytes.Buffer + r := &runtime{ + ctx: context.Background(), + cfg: config.Default(), + stdout: &stdout, + } + r.cfg.Search.Lexical.Languages = []string{"ko", "zh"} + r.cfg.Search.Lexical.Python = "/tmp/tokenizers/bin/python" + r.installLexical = func( + _ context.Context, + python string, + languages []string, + ) (store.LexicalInstallResult, error) { + require.Equal(t, "/tmp/tokenizers/bin/python", python) + require.Equal(t, []string{"ko", "zh"}, languages) + return store.LexicalInstallResult{ + Packages: []string{"kiwipiepy==0.23.2", "jieba==0.42.1"}, + }, nil + } + + require.NoError(t, r.runLexical([]string{"install"})) + require.Contains(t, stdout.String(), "kiwipiepy==0.23.2") + require.Contains(t, stdout.String(), "jieba==0.42.1") +} + +func TestRunLexicalInstallRequiresConfiguredLanguages(t *testing.T) { + r := &runtime{ + ctx: context.Background(), + cfg: config.Default(), + installLexical: func(context.Context, string, []string) (store.LexicalInstallResult, error) { + return store.LexicalInstallResult{}, errors.New("must not run") + }, + } + + err := r.runLexical([]string{"install"}) + require.ErrorContains(t, err, "search.lexical.languages is empty") +} + +func TestRunLexicalInstallReportsInstallerError(t *testing.T) { + r := &runtime{ + ctx: context.Background(), + cfg: config.Default(), + installLexical: func(context.Context, string, []string) (store.LexicalInstallResult, error) { + return store.LexicalInstallResult{}, errors.New("pip unavailable") + }, + } + r.cfg.Search.Lexical.Languages = []string{"ko"} + + err := r.runLexical([]string{"install"}) + require.ErrorContains(t, err, "pip unavailable") + err = r.runLexical([]string{"unknown"}) + require.ErrorContains(t, err, "usage: discrawl lexical install") +} + +func TestRunLexicalInstallJSONOutput(t *testing.T) { + var stdout bytes.Buffer + r := &runtime{ + ctx: context.Background(), + cfg: config.Default(), + stdout: &stdout, + json: true, + installLexical: func(context.Context, string, []string) (store.LexicalInstallResult, error) { + return store.LexicalInstallResult{Packages: []string{"kiwipiepy==0.23.2"}}, nil + }, + } + r.cfg.Search.Lexical.Languages = []string{"ko"} + + require.NoError(t, r.runLexical([]string{"install"})) + require.JSONEq(t, `{ + "languages": ["ko"], + "packages": ["kiwipiepy==0.23.2"], + "python": "python3" + }`, stdout.String()) +} + +func TestLexicalHelp(t *testing.T) { + var stdout bytes.Buffer + require.NoError(t, Run(context.Background(), []string{"help", "lexical"}, &stdout, &bytes.Buffer{})) + require.Contains(t, stdout.String(), "discrawl lexical install") +} + +func TestRunDispatchesLexicalInstall(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.toml") + require.NoError(t, os.WriteFile(configPath, []byte(` +version = 1 + +[discord] +token_source = "env" +`), 0o600)) + t.Setenv("DISCORD_BOT_TOKEN", "dummy") + + err := Run( + context.Background(), + []string{"--config", configPath, "lexical", "install"}, + &bytes.Buffer{}, + &bytes.Buffer{}, + ) + require.ErrorContains(t, err, "search.lexical.languages is empty") +} diff --git a/internal/cli/output.go b/internal/cli/output.go index 427ba23f..610f0710 100644 --- a/internal/cli/output.go +++ b/internal/cli/output.go @@ -248,6 +248,12 @@ Flags: --dm Search local desktop DM cache. --guild ID Restrict to one guild id. --guilds ID,ID Restrict to guild ids. +`, + "lexical": `Usage: + discrawl lexical install + +Install pinned tokenizer packages for the languages in search.lexical.languages. +The configured Python interpreter must belong to a virtual environment. `, "attachments": `Usage: discrawl attachments [flags] diff --git a/internal/store/lexical.go b/internal/store/lexical.go index 25b8eae3..86b8417d 100644 --- a/internal/store/lexical.go +++ b/internal/store/lexical.go @@ -41,6 +41,10 @@ func openWithLexicalTokenizers( _ = store.Close() return nil, err } + if err := store.invalidateDisabledLexicalVersions(ctx); err != nil { + _ = store.Close() + return nil, err + } return store, nil } @@ -96,6 +100,66 @@ func (s *Store) ensureLexicalFTS(ctx context.Context) error { return nil } +func (s *Store) invalidateDisabledLexicalVersions(ctx context.Context) error { + enabled := make(map[string]struct{}, len(s.lexicalTokenizers)) + for language := range s.lexicalTokenizers { + enabled[language] = struct{}{} + } + knownScopes := map[string]string{ + lexicalFTSScope("ko"): "ko", + lexicalFTSScope("ja"): "ja", + lexicalFTSScope("zh"): "zh", + lexicalFTSScope("ar"): "ar", + } + disabledScopes, err := func() ([]string, error) { + rows, err := s.db.QueryContext(ctx, ` + select scope + from sync_state + where scope in (?, ?, ?, ?) + `, + lexicalFTSScope("ko"), + lexicalFTSScope("ja"), + lexicalFTSScope("zh"), + lexicalFTSScope("ar"), + ) + if err != nil { + return nil, fmt.Errorf("query lexical index versions: %w", err) + } + defer func() { _ = rows.Close() }() + var scopes []string + for rows.Next() { + var scope string + if err := rows.Scan(&scope); err != nil { + return nil, fmt.Errorf("scan lexical index version: %w", err) + } + language, ok := knownScopes[scope] + if !ok { + continue + } + if _, ok := enabled[language]; !ok { + scopes = append(scopes, scope) + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate lexical index versions: %w", err) + } + return scopes, nil + }() + if err != nil { + return err + } + for _, scope := range disabledScopes { + if _, err := s.db.ExecContext( + ctx, + `delete from sync_state where scope = ?`, + scope, + ); err != nil { + return fmt.Errorf("invalidate disabled lexical index %s: %w", scope, err) + } + } + return nil +} + func (s *Store) rebuildLexicalIndexes(ctx context.Context) error { for _, language := range s.lexicalLanguages() { if err := s.rebuildLexicalFTS(ctx, language); err != nil { diff --git a/internal/store/lexical_install.go b/internal/store/lexical_install.go new file mode 100644 index 00000000..990dd4d9 --- /dev/null +++ b/internal/store/lexical_install.go @@ -0,0 +1,131 @@ +package store + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "runtime" + "strings" +) + +type LexicalInstallResult struct { + Packages []string + Output string +} + +type lexicalCommandRunner func(context.Context, string, ...string) ([]byte, error) + +func InstallLexicalPackages( + ctx context.Context, + python string, + languages []string, +) (LexicalInstallResult, error) { + return installLexicalPackagesWithRunner(ctx, python, languages, runLexicalCommand) +} + +func installLexicalPackagesWithRunner( + ctx context.Context, + python string, + languages []string, + run lexicalCommandRunner, +) (LexicalInstallResult, error) { + packages, err := lexicalPythonPackages(languages) + if err != nil { + return LexicalInstallResult{}, err + } + if len(packages) == 0 { + return LexicalInstallResult{}, errors.New("no search.lexical languages are configured") + } + python, err = expandLexicalPython(python) + if err != nil { + return LexicalInstallResult{}, err + } + spec, err := pythonCommandSpec(python, runtime.GOOS == "windows") + if err != nil { + return LexicalInstallResult{}, err + } + probeArgs := append([]string{}, spec.ArgsPrefix...) + probeArgs = append(probeArgs, "-c", "import sys; print(sys.prefix != sys.base_prefix)") + output, err := run(ctx, spec.Path, probeArgs...) + if err != nil { + return LexicalInstallResult{}, fmt.Errorf("check lexical Python virtual environment: %w", err) + } + if !strings.EqualFold(strings.TrimSpace(string(output)), "true") { + return LexicalInstallResult{}, errors.New( + "lexical package installation requires a virtual environment; configure search.lexical.python to its interpreter", + ) + } + installArgs := append([]string{}, spec.ArgsPrefix...) + installArgs = append( + installArgs, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-input", + "--require-virtualenv", + ) + installArgs = append(installArgs, packages...) + output, err = run(ctx, spec.Path, installArgs...) + if err != nil { + return LexicalInstallResult{}, fmt.Errorf("install lexical Python packages: %w", err) + } + return LexicalInstallResult{ + Packages: packages, + Output: strings.TrimSpace(string(output)), + }, nil +} + +func lexicalPythonPackages(languages []string) ([]string, error) { + packages := make([]string, 0, len(languages)+1) + seen := make(map[string]struct{}, len(languages)) + for _, language := range languages { + if _, ok := seen[language]; ok { + continue + } + seen[language] = struct{}{} + switch language { + case "ko": + packages = append(packages, "kiwipiepy==0.23.2") + case "ja": + packages = append(packages, "sudachipy==0.6.11", "sudachidict_core==20260723") + case "zh": + packages = append(packages, "jieba==0.42.1") + case "ar": + packages = append(packages, "snowballstemmer==3.1.1") + default: + return nil, fmt.Errorf("unsupported lexical language %q", language) + } + } + return packages, nil +} + +func runLexicalCommand(ctx context.Context, path string, args ...string) ([]byte, error) { + var output bytes.Buffer + cmd := &exec.Cmd{ + Path: path, + Args: append([]string{path}, args...), + Stdout: &output, + Stderr: &output, + } + if err := cmd.Start(); err != nil { + return nil, err + } + wait := make(chan error, 1) + go func() { + wait <- cmd.Wait() + }() + select { + case err := <-wait: + if err != nil { + return output.Bytes(), fmt.Errorf("%w: %s", err, strings.TrimSpace(output.String())) + } + return output.Bytes(), nil + case <-ctx.Done(): + _ = cmd.Process.Kill() + <-wait + return output.Bytes(), ctx.Err() + } +} diff --git a/internal/store/lexical_lazy.go b/internal/store/lexical_lazy.go new file mode 100644 index 00000000..04dcbb91 --- /dev/null +++ b/internal/store/lexical_lazy.go @@ -0,0 +1,34 @@ +package store + +import ( + "context" + "sync" +) + +type lazyLexicalTokenizer struct { + start func() (LexicalTokenizer, error) + once sync.Once + tokenizer LexicalTokenizer + err error +} + +func newLazyLexicalTokenizer(start func() (LexicalTokenizer, error)) LexicalTokenizer { + return &lazyLexicalTokenizer{start: start} +} + +func (l *lazyLexicalTokenizer) Tokenize(ctx context.Context, text string) (string, error) { + l.once.Do(func() { + l.tokenizer, l.err = l.start() + }) + if l.err != nil { + return "", l.err + } + return l.tokenizer.Tokenize(ctx, text) +} + +func (l *lazyLexicalTokenizer) Close() error { + if l == nil || l.tokenizer == nil { + return nil + } + return l.tokenizer.Close() +} diff --git a/internal/store/lexical_lazy_test.go b/internal/store/lexical_lazy_test.go new file mode 100644 index 00000000..729bf26f --- /dev/null +++ b/internal/store/lexical_lazy_test.go @@ -0,0 +1,50 @@ +package store + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLazyLexicalTokenizerStartsOnce(t *testing.T) { + starts := 0 + tokenizer := newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + starts++ + return stubLexicalTokenizer{tokenize: func(text string) string { + return text + " tokenized" + }}, nil + }) + + first, err := tokenizer.Tokenize(context.Background(), "first") + require.NoError(t, err) + require.Equal(t, "first tokenized", first) + second, err := tokenizer.Tokenize(context.Background(), "second") + require.NoError(t, err) + require.Equal(t, "second tokenized", second) + require.Equal(t, 1, starts) + require.NoError(t, tokenizer.Close()) +} + +func TestLazyLexicalTokenizerCachesStartupFailure(t *testing.T) { + starts := 0 + tokenizer := newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + starts++ + return nil, errors.New("startup failed") + }) + + _, err := tokenizer.Tokenize(context.Background(), "first") + require.ErrorContains(t, err, "startup failed") + _, err = tokenizer.Tokenize(context.Background(), "second") + require.ErrorContains(t, err, "startup failed") + require.Equal(t, 1, starts) + require.NoError(t, tokenizer.Close()) +} + +func TestLazyLexicalTokenizerCloseBeforeStart(t *testing.T) { + tokenizer := newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + return nil, errors.New("must not start") + }) + require.NoError(t, tokenizer.Close()) +} diff --git a/internal/store/lexical_python.go b/internal/store/lexical_python.go index f53135db..b603d9be 100644 --- a/internal/store/lexical_python.go +++ b/internal/store/lexical_python.go @@ -73,29 +73,34 @@ func newPythonLexicalTokenizers(opts OpenOptions) (map[string]LexicalTokenizer, if len(opts.LexicalLanguages) == 0 { return nil, nil } - python := strings.TrimSpace(opts.LexicalPython) - if python == "" { - python = "python3" - } - if strings.HasPrefix(python, "~/") { - home, err := os.UserHomeDir() - if err != nil { - return nil, fmt.Errorf("resolve lexical Python home directory: %w", err) - } - python = filepath.Join(home, strings.TrimPrefix(python, "~/")) + python, err := expandLexicalPython(opts.LexicalPython) + if err != nil { + return nil, err } tokenizers := make(map[string]LexicalTokenizer, len(opts.LexicalLanguages)) for _, language := range opts.LexicalLanguages { - tokenizer, err := startPythonLexicalTokenizer(python, language) - if err != nil { - closeLexicalTokenizers(tokenizers) - return nil, err - } - tokenizers[language] = tokenizer + tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + return startPythonLexicalTokenizer(python, language) + }) } return tokenizers, nil } +func expandLexicalPython(python string) (string, error) { + python = strings.TrimSpace(python) + if python == "" { + return "python3", nil + } + if !strings.HasPrefix(python, "~/") { + return python, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolve lexical Python home directory: %w", err) + } + return filepath.Join(home, strings.TrimPrefix(python, "~/")), nil +} + func startPythonLexicalTokenizer( python string, language string, diff --git a/internal/store/lexical_python_command.go b/internal/store/lexical_python_command.go new file mode 100644 index 00000000..1cf1d7cc --- /dev/null +++ b/internal/store/lexical_python_command.go @@ -0,0 +1,102 @@ +package store + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +type lexicalCommandSpec struct { + Path string + ArgsPrefix []string +} + +func pythonCommandSpec(python string, windows bool) (lexicalCommandSpec, error) { + python = strings.TrimSpace(python) + if python == "" { + python = "python3" + } + if isAbsolutePythonPath(python, windows) { + return lexicalCommandSpec{Path: python}, nil + } + allowed := map[string][]string{ + "python": nil, + "python3": nil, + } + if windows { + allowed["python.exe"] = nil + allowed["python3.exe"] = nil + allowed["py"] = []string{"-3"} + allowed["py.exe"] = []string{"-3"} + } + prefix, ok := allowed[python] + if !ok { + return lexicalCommandSpec{}, fmt.Errorf( + "unsupported lexical Python interpreter %q; use an absolute path or python/python3", + python, + ) + } + path, err := exec.LookPath(python) + if err != nil { + return lexicalCommandSpec{}, fmt.Errorf("find lexical Python interpreter %q: %w", python, err) + } + return lexicalCommandSpec{Path: path, ArgsPrefix: prefix}, nil +} + +func newPythonLexicalCommand(python string, language string) (*exec.Cmd, error) { + spec, err := pythonCommandSpec(python, runtime.GOOS == "windows") + if err != nil { + return nil, err + } + args := append([]string{spec.Path}, spec.ArgsPrefix...) + args = append(args, "-u", "-c", pythonLexicalWorker, language) + return &exec.Cmd{ + Path: spec.Path, + Args: args, + Env: lexicalWorkerEnvironment(os.Environ()), + }, nil +} + +func isAbsolutePythonPath(path string, windows bool) bool { + if windows { + if strings.HasPrefix(path, `\\`) { + return true + } + return len(path) >= 3 && + ((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] <= 'z')) && + path[1] == ':' && + (path[2] == '\\' || path[2] == '/') + } + return filepath.IsAbs(path) +} + +func lexicalWorkerEnvironment(parent []string) []string { + allowed := map[string]struct{}{ + "HOME": {}, + "LANG": {}, + "LC_ALL": {}, + "PATH": {}, + "PATHEXT": {}, + "SYSTEMROOT": {}, + "TEMP": {}, + "TMP": {}, + "TMPDIR": {}, + "VIRTUAL_ENV": {}, + "WINDIR": {}, + } + environment := make([]string, 0, len(allowed)+2) + for _, entry := range parent { + key, _, ok := strings.Cut(entry, "=") + if !ok { + continue + } + if _, ok := allowed[strings.ToUpper(key)]; ok { + environment = append(environment, entry) + } + } + environment = append(environment, "PYTHONNOUSERSITE=1", "PYTHONUTF8=1") + return environment +} diff --git a/internal/store/lexical_python_command_unix.go b/internal/store/lexical_python_command_unix.go deleted file mode 100644 index 5dab4f33..00000000 --- a/internal/store/lexical_python_command_unix.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build !windows - -package store - -import ( - "os/exec" -) - -func newPythonLexicalCommand( - python string, - language string, -) (*exec.Cmd, error) { - return &exec.Cmd{ - Path: "/usr/bin/env", - Args: []string{ - "/usr/bin/env", - python, - "-u", - "-c", - pythonLexicalWorker, - language, - }, - }, nil -} diff --git a/internal/store/lexical_python_command_windows.go b/internal/store/lexical_python_command_windows.go deleted file mode 100644 index 304f0965..00000000 --- a/internal/store/lexical_python_command_windows.go +++ /dev/null @@ -1,28 +0,0 @@ -//go:build windows - -package store - -import ( - "fmt" - "os/exec" -) - -func newPythonLexicalCommand( - python string, - language string, -) (*exec.Cmd, error) { - args := []string{"-u", "-c", pythonLexicalWorker, language} - switch python { - case "python", "python.exe": - return exec.Command("python", args...), nil - case "python3", "python3.exe": - return exec.Command("python3", args...), nil - case "py", "py.exe": - return exec.Command("py", append([]string{"-3"}, args...)...), nil - default: - return nil, fmt.Errorf( - "unsupported Windows lexical Python launcher %q; use python, python3, or py", - python, - ) - } -} diff --git a/internal/store/lexical_python_test.go b/internal/store/lexical_python_test.go index e4991912..dad68d5a 100644 --- a/internal/store/lexical_python_test.go +++ b/internal/store/lexical_python_test.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" ) @@ -26,6 +27,14 @@ func TestPythonLexicalTokenizerCommandProtocol(t *testing.T) { require.NoError(t, tokenizer.Close()) } +func TestLockedBuffer(t *testing.T) { + var buffer lockedBuffer + written, err := buffer.Write([]byte("tokenizer stderr")) + require.NoError(t, err) + require.Equal(t, len("tokenizer stderr"), written) + require.Equal(t, "tokenizer stderr", buffer.String()) +} + func TestPythonLexicalTokenizerCommandStartupError(t *testing.T) { tokenizer, err := startPythonLexicalTokenizerCommand( lexicalHelperCommand("startup-error"), @@ -35,6 +44,24 @@ func TestPythonLexicalTokenizerCommandStartupError(t *testing.T) { require.ErrorContains(t, err, "missing tokenizer package") } +func TestPythonLexicalTokenizerCommandMalformedStartup(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizerCommand( + lexicalHelperCommand("malformed-startup"), + "test", + ) + require.Nil(t, tokenizer) + require.ErrorContains(t, err, "decode tokenizer response") +} + +func TestPythonLexicalTokenizerCommandStartupStderr(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizerCommand( + lexicalHelperCommand("stderr-startup"), + "test", + ) + require.Nil(t, tokenizer) + require.ErrorContains(t, err, "tokenizer stderr") +} + func TestPythonLexicalTokenizerCommandResponseError(t *testing.T) { tokenizer, err := startPythonLexicalTokenizerCommand( lexicalHelperCommand("response-error"), @@ -61,6 +88,18 @@ func TestPythonLexicalTokenizerHonorsCanceledContext(t *testing.T) { require.ErrorIs(t, err, context.Canceled) } +func TestPythonLexicalTokenizerReportsWriteAfterClose(t *testing.T) { + tokenizer, err := startPythonLexicalTokenizerCommand( + lexicalHelperCommand("ready"), + "test", + ) + require.NoError(t, err) + require.NoError(t, tokenizer.Close()) + + _, err = tokenizer.Tokenize(context.Background(), "mixed text") + require.ErrorContains(t, err, "write tokenizer request") +} + func TestNewPythonLexicalTokenizersDisabled(t *testing.T) { tokenizers, err := newPythonLexicalTokenizers(OpenOptions{}) require.NoError(t, err) @@ -93,12 +132,21 @@ func TestNewPythonLexicalTokenizersExpandsHomePath(t *testing.T) { closeLexicalTokenizers(tokenizers) } -func TestOpenWithOptionsReportsMissingPython(t *testing.T) { - _, err := OpenWithOptions(context.Background(), filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ +func TestOpenWithOptionsLoadsPythonLazily(t *testing.T) { + ctx := context.Background() + s, err := OpenWithOptions(ctx, filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ LexicalLanguages: []string{"ko"}, LexicalPython: "/definitely/missing/discrawl-python", }) - require.ErrorContains(t, err, "initialize ko lexical tokenizer") + require.NoError(t, err) + defer func() { _ = s.Close() }() + + err = s.UpsertMessage(ctx, MessageRecord{ + ID: "ko", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + }) + require.ErrorContains(t, err, "start ko lexical tokenizer") } func lexicalHelperCommand(mode string) *exec.Cmd { @@ -118,6 +166,14 @@ func TestLexicalTokenizerHelperProcess(t *testing.T) { fmt.Println(`{"error":"missing tokenizer package"}`) os.Exit(2) } + if mode == "malformed-startup" { + fmt.Println(`not-json`) + os.Exit(2) + } + if mode == "stderr-startup" { + _, _ = fmt.Fprintln(os.Stderr, "tokenizer stderr") + os.Exit(2) + } fmt.Println(`{"ready":true}`) scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { diff --git a/internal/store/lexical_review_test.go b/internal/store/lexical_review_test.go new file mode 100644 index 00000000..d7b72523 --- /dev/null +++ b/internal/store/lexical_review_test.go @@ -0,0 +1,218 @@ +package store + +import ( + "context" + "errors" + "fmt" + "os" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLexicalPythonPackagesSelectOnlyConfiguredLanguages(t *testing.T) { + packages, err := lexicalPythonPackages([]string{"ko", "zh"}) + require.NoError(t, err) + require.Equal(t, []string{ + "kiwipiepy==0.23.2", + "jieba==0.42.1", + }, packages) + require.NotContains(t, packages, "sudachipy==0.6.11") + require.NotContains(t, packages, "snowballstemmer==3.1.1") +} + +func TestLexicalPythonPackagesRejectsEmptyAndUnknownLanguages(t *testing.T) { + packages, err := lexicalPythonPackages(nil) + require.NoError(t, err) + require.Empty(t, packages) + + _, err = lexicalPythonPackages([]string{"ko", "unknown"}) + require.ErrorContains(t, err, `unsupported lexical language "unknown"`) +} + +func TestInstallLexicalPackagesRejectsNoConfiguredLanguages(t *testing.T) { + _, err := InstallLexicalPackages(context.Background(), "python3", nil) + require.ErrorContains(t, err, "no search.lexical languages") +} + +func TestInstallLexicalPackagesRequiresVirtualEnvironment(t *testing.T) { + _, err := installLexicalPackagesWithRunner( + context.Background(), + "/tmp/python", + []string{"ko"}, + func(context.Context, string, ...string) ([]byte, error) { + return []byte("false\n"), nil + }, + ) + require.ErrorContains(t, err, "virtual environment") +} + +func TestInstallLexicalPackagesUsesPinnedSelectedPackages(t *testing.T) { + var commands [][]string + result, err := installLexicalPackagesWithRunner( + context.Background(), + "/tmp/python", + []string{"ja", "ar"}, + func(_ context.Context, path string, args ...string) ([]byte, error) { + commands = append(commands, append([]string{path}, args...)) + if slices.Contains(args, "import sys; print(sys.prefix != sys.base_prefix)") { + return []byte("true\n"), nil + } + return []byte("installed\n"), nil + }, + ) + require.NoError(t, err) + require.Equal(t, []string{ + "sudachipy==0.6.11", + "sudachidict_core==20260723", + "snowballstemmer==3.1.1", + }, result.Packages) + require.Len(t, commands, 2) + require.Equal(t, []string{ + "/tmp/python", + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-input", + "--require-virtualenv", + "sudachipy==0.6.11", + "sudachidict_core==20260723", + "snowballstemmer==3.1.1", + }, commands[1]) +} + +func TestInstallLexicalPackagesReportsBoundaryFailures(t *testing.T) { + _, err := installLexicalPackagesWithRunner( + context.Background(), + "/tmp/python", + nil, + func(context.Context, string, ...string) ([]byte, error) { + return nil, errors.New("must not run") + }, + ) + require.ErrorContains(t, err, "no search.lexical languages") + + _, err = installLexicalPackagesWithRunner( + context.Background(), + "sh", + []string{"ko"}, + func(context.Context, string, ...string) ([]byte, error) { + return nil, errors.New("must not run") + }, + ) + require.ErrorContains(t, err, "unsupported lexical Python interpreter") + + _, err = installLexicalPackagesWithRunner( + context.Background(), + "/tmp/python", + []string{"ko"}, + func(context.Context, string, ...string) ([]byte, error) { + return nil, errors.New("probe failed") + }, + ) + require.ErrorContains(t, err, "check lexical Python virtual environment") + + calls := 0 + _, err = installLexicalPackagesWithRunner( + context.Background(), + "/tmp/python", + []string{"ko"}, + func(context.Context, string, ...string) ([]byte, error) { + calls++ + if calls == 1 { + return []byte("True\n"), nil + } + return []byte("pip failed"), errors.New("install failed") + }, + ) + require.ErrorContains(t, err, "install lexical Python packages") +} + +func TestPythonCommandSpecAllowsKnownLaunchersAndAbsolutePaths(t *testing.T) { + spec, err := pythonCommandSpec("python3", false) + require.NoError(t, err) + require.NotEmpty(t, spec.Path) + + spec, err = pythonCommandSpec("/opt/discrawl/tokenizers/bin/python", false) + require.NoError(t, err) + require.Equal(t, "/opt/discrawl/tokenizers/bin/python", spec.Path) + + spec, err = pythonCommandSpec(`C:\discrawl-tokenizers\Scripts\python.exe`, true) + require.NoError(t, err) + require.Equal(t, `C:\discrawl-tokenizers\Scripts\python.exe`, spec.Path) + + _, err = pythonCommandSpec("sh", false) + require.ErrorContains(t, err, "unsupported lexical Python interpreter") + + require.True(t, isAbsolutePythonPath(`\\server\share\python.exe`, true)) + require.True(t, isAbsolutePythonPath(`D:/venv/python.exe`, true)) + require.False(t, isAbsolutePythonPath(`venv\python.exe`, true)) + require.False(t, isAbsolutePythonPath("venv/python", false)) +} + +func TestLexicalWorkerEnvironmentDropsParentSecrets(t *testing.T) { + environment := lexicalWorkerEnvironment([]string{ + "PATH=/usr/bin", + "HOME=/tmp/home", + "LANG=en_US.UTF-8", + "VIRTUAL_ENV=/tmp/venv", + "DISCORD_BOT_TOKEN=secret", + "OPENAI_API_KEY=secret", + "PIP_INDEX_URL=https://user:password@example.invalid/simple", + }) + joined := strings.Join(environment, "\n") + require.Contains(t, joined, "PATH=/usr/bin") + require.Contains(t, joined, "HOME=/tmp/home") + require.Contains(t, joined, "LANG=en_US.UTF-8") + require.Contains(t, joined, "VIRTUAL_ENV=/tmp/venv") + require.Contains(t, joined, "PYTHONNOUSERSITE=1") + require.NotContains(t, joined, "DISCORD_BOT_TOKEN") + require.NotContains(t, joined, "OPENAI_API_KEY") + require.NotContains(t, joined, "PIP_INDEX_URL") + require.NotContains(t, joined, "password") +} + +func TestRunLexicalCommandCapturesOutputAndFailure(t *testing.T) { + t.Setenv("DISCRAWL_INSTALL_HELPER", "1") + output, err := runLexicalCommand( + context.Background(), + os.Args[0], + "-test.run=TestLexicalInstallCommandHelperProcess", + "--", + "success", + ) + require.NoError(t, err) + require.Equal(t, "installed\n", string(output)) + + output, err = runLexicalCommand( + context.Background(), + os.Args[0], + "-test.run=TestLexicalInstallCommandHelperProcess", + "--", + "failure", + ) + require.ErrorContains(t, err, "exit status") + require.Contains(t, string(output), "install failed") + + _, err = runLexicalCommand(context.Background(), "/definitely/missing/discrawl-command") + require.Error(t, err) +} + +func TestLexicalInstallCommandHelperProcess(t *testing.T) { + if os.Getenv("DISCRAWL_INSTALL_HELPER") != "1" { + return + } + switch os.Args[len(os.Args)-1] { + case "success": + fmt.Println("installed") + os.Exit(0) + case "failure": + fmt.Println("install failed") + os.Exit(2) + default: + os.Exit(3) + } +} diff --git a/internal/store/multilingual_lifecycle_test.go b/internal/store/multilingual_lifecycle_test.go index 1cf30575..03e7b50d 100644 --- a/internal/store/multilingual_lifecycle_test.go +++ b/internal/store/multilingual_lifecycle_test.go @@ -97,6 +97,44 @@ func TestMultilingualIndexVersionSurvivesReopen(t *testing.T) { require.Equal(t, []string{"ko"}, searchResultIDs(results)) } +func TestMultilingualIndexRebuildsAfterDisabledWrites(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "discrawl.db") + tokenizers := func() map[string]LexicalTokenizer { + return map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "저녁먹음": "저녁 먹 음", + "회의기록": "회의 기록", + })}, + } + } + + enabled, err := openWithLexicalTokenizers(ctx, path, tokenizers()) + require.NoError(t, err) + require.NoError(t, enabled.UpsertMessage(ctx, MessageRecord{ + ID: "before", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + })) + require.NoError(t, enabled.Close()) + + disabled, err := openWithLexicalTokenizers(ctx, path, nil) + require.NoError(t, err) + require.NoError(t, disabled.UpsertMessage(ctx, MessageRecord{ + ID: "during", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Add(time.Minute).Format(time.RFC3339Nano), + Content: "회의기록", NormalizedContent: "회의기록", RawJSON: `{}`, + })) + require.NoError(t, disabled.Close()) + + reenabled, err := openWithLexicalTokenizers(ctx, path, tokenizers()) + require.NoError(t, err) + defer func() { _ = reenabled.Close() }() + results, err := reenabled.SearchMessages(ctx, SearchOptions{Query: "기록", Limit: 10}) + require.NoError(t, err) + require.Equal(t, []string{"during"}, searchResultIDs(results)) +} + func TestMultilingualIndexesSearchThroughReadOnlyStore(t *testing.T) { ctx := context.Background() path := filepath.Join(t.TempDir(), "discrawl.db") @@ -124,6 +162,33 @@ func TestMultilingualIndexesSearchThroughReadOnlyStore(t *testing.T) { require.Equal(t, []string{"ko"}, searchResultIDs(results)) } +func TestOpenReadOnlyWithOptionsWithoutLexicalLanguages(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "discrawl.db") + writer, err := Open(ctx, path) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + reader, err := OpenReadOnlyWithOptions(ctx, path, OpenOptions{}) + require.NoError(t, err) + require.NoError(t, reader.Close()) +} + +func TestOpenReadOnlyWithOptionsKeepsMissingTokenizerLazy(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "discrawl.db") + writer, err := Open(ctx, path) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + reader, err := OpenReadOnlyWithOptions(ctx, path, OpenOptions{ + LexicalLanguages: []string{"ko"}, + LexicalPython: "/definitely/missing/discrawl-python", + }) + require.NoError(t, err) + require.NoError(t, reader.Close()) +} + func TestMultilingualTokenizerFailureAbortsWrite(t *testing.T) { ctx := context.Background() s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), nil) From 89ea8a19d6b12be69491c9fe15da6122ff63922c Mon Sep 17 00:00:00 2001 From: "Jeffrey (Dongkyu) Kim" Date: Wed, 19 Aug 2026 13:59:07 +0900 Subject: [PATCH 3/5] refactor(search): use kiwigo for Korean tokenization --- .github/workflows/ci.yml | 27 +++ THIRD_PARTY_NOTICES.md | 18 ++ docs/benchmarks/multilingual-lexical.md | 20 +- docs/commands/lexical.md | 25 ++- docs/configuration.md | 9 +- docs/guides/search-modes.md | 31 ++- internal/cli/cli.go | 14 +- internal/cli/lexical_commands.go | 4 +- internal/cli/lexical_commands_test.go | 10 +- internal/cli/output.go | 4 +- internal/config/config.go | 14 +- internal/config/config_test.go | 4 + internal/store/lexical_install.go | 9 +- internal/store/lexical_kiwi.go | 211 ++++++++++++++++++ internal/store/lexical_python.go | 36 +-- internal/store/lexical_python_test.go | 27 ++- internal/store/lexical_review_test.go | 142 +++++++++++- internal/store/multilingual_benchmark_test.go | 6 +- ...n_e2e_test.go => multilingual_e2e_test.go} | 8 +- internal/store/multilingual_lifecycle_test.go | 5 +- tools/discrawl-kiwi/README.md | 47 ++++ tools/discrawl-kiwi/go.mod | 14 ++ tools/discrawl-kiwi/go.sum | 15 ++ tools/discrawl-kiwi/install-kiwi.sh | 33 +++ tools/discrawl-kiwi/main.go | 95 ++++++++ tools/discrawl-kiwi/main_test.go | 40 ++++ 26 files changed, 792 insertions(+), 76 deletions(-) create mode 100644 THIRD_PARTY_NOTICES.md create mode 100644 internal/store/lexical_kiwi.go rename internal/store/{multilingual_python_e2e_test.go => multilingual_e2e_test.go} (84%) create mode 100644 tools/discrawl-kiwi/README.md create mode 100644 tools/discrawl-kiwi/go.mod create mode 100644 tools/discrawl-kiwi/go.sum create mode 100644 tools/discrawl-kiwi/install-kiwi.sh create mode 100644 tools/discrawl-kiwi/main.go create mode 100644 tools/discrawl-kiwi/main_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6fe6162..fb45d5b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,33 @@ jobs: version: "~> v2" args: release --snapshot --clean --skip=publish + kiwi-helper: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: tools/discrawl-kiwi/go.mod + cache-dependency-path: tools/discrawl-kiwi/go.sum + + - name: Install Kiwi 0.23.2 + working-directory: tools/discrawl-kiwi + run: bash install-kiwi.sh + + - name: Test Go binding helper + working-directory: tools/discrawl-kiwi + run: go test -count=1 ./... + secrets: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 00000000..fd89caaa --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,18 @@ +# Third-party notices + +## Optional Korean lexical helper + +The separately built `tools/discrawl-kiwi` helper depends on: + +- [Kiwi](https://github.com/bab2min/Kiwi), copyright Minchul Lee, + licensed under GNU LGPL 2.1 or later. +- [github.com/codingpot/kiwigo](https://github.com/codingpot/kiwigo), a Go + binding for Kiwi, licensed under GNU LGPL 2.1. + +These dependencies are optional and are not linked into the default Discrawl +binary. Distributors who provide the helper or Kiwi native binaries must +satisfy their applicable LGPL notice, source-access, and relinking +requirements. Kiwi's full license text is available from its source +repository and the GNU project: + +https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html diff --git a/docs/benchmarks/multilingual-lexical.md b/docs/benchmarks/multilingual-lexical.md index 82ce5a89..7cfdc87d 100644 --- a/docs/benchmarks/multilingual-lexical.md +++ b/docs/benchmarks/multilingual-lexical.md @@ -13,7 +13,6 @@ cannot retrieve as independent terms. ```bash python3 -m venv /tmp/discrawl-tokenizer-e2e /tmp/discrawl-tokenizer-e2e/bin/python -m pip install \ - kiwipiepy==0.23.2 \ sudachipy==0.6.11 \ sudachidict_core==20260723 \ jieba==0.42.1 \ @@ -21,6 +20,8 @@ python3 -m venv /tmp/discrawl-tokenizer-e2e DISCRAWL_TOKENIZER_E2E=1 \ DISCRAWL_TOKENIZER_PYTHON=/tmp/discrawl-tokenizer-e2e/bin/python \ +DISCRAWL_KIWI_HELPER=/tmp/discrawl-kiwi \ +DISCRAWL_KIWI_MODEL=/tmp/kiwi-model/models/cong/base \ go test ./internal/store \ -run TestMultilingualLexicalQualityBenchmark \ -count=1 -v @@ -28,7 +29,8 @@ go test ./internal/store \ ## Result -Measured on macOS arm64 with Python 3.13.2: +Measured on macOS arm64 with Kiwi 0.23.2 through `kiwigo`; the remaining +analyzers used Python 3.13.2: | Language | `unicode61` recall@5 | Multilingual recall@5 | | --- | ---: | ---: | @@ -59,13 +61,13 @@ ja recall@5: unicode61=0/5 multilingual=5/5 zh recall@5: unicode61=0/5 multilingual=5/5 ar recall@5: unicode61=0/5 multilingual=5/5 --- PASS: TestMultilingualLexicalQualityBenchmark -=== RUN TestPythonMultilingualLexicalSearchE2E ---- PASS: TestPythonMultilingualLexicalSearchE2E +=== RUN TestMultilingualLexicalSearchE2E +--- PASS: TestMultilingualLexicalSearchE2E PASS ``` -The selected-install path was separately exercised from an empty virtual -environment with `languages = ["ko"]`. `discrawl lexical install` installed -`kiwipiepy==0.23.2` plus its runtime dependencies, did not install Sudachi, -Jieba, or Snowball, and a subsequent CLI search for `저녁` returned the fixture -`오늘 저녁먹음 기록`. +The Korean path now uses the published `github.com/codingpot/kiwigo` Go binding +against Kiwi 0.23.2. The live helper returned `오늘 저녁 먹 음 기록` for +`오늘 저녁먹음 기록`; a subsequent CLI search for `저녁` returned that fixture. +No Python process or `kiwipiepy` package is involved in Korean indexing or +query analysis. diff --git a/docs/commands/lexical.md b/docs/commands/lexical.md index a077afc0..b28806dd 100644 --- a/docs/commands/lexical.md +++ b/docs/commands/lexical.md @@ -1,7 +1,8 @@ # `lexical` -Installs only the optional tokenizer packages selected by -`search.lexical.languages`. +Installs only the optional Python tokenizer packages selected by +`search.lexical.languages`. Korean is provided by the native +`discrawl-kiwi` Go helper and is not installed by this command. ## Usage @@ -10,13 +11,15 @@ discrawl lexical install ``` The configured `search.lexical.python` interpreter must belong to a virtual -environment. Discrawl refuses to install packages into a system Python. +environment when Japanese, Chinese, or Arabic is selected. Discrawl refuses to +install packages into a system Python. With a Korean-only configuration this +command is a no-op because Korean uses the separately built Go helper. Package versions are pinned by Discrawl: | Language | Packages | | --- | --- | -| `ko` | `kiwipiepy==0.23.2` | +| `ko` | none; uses `github.com/codingpot/kiwigo` + Kiwi 0.23.2 | | `ja` | `sudachipy==0.6.11`, `sudachidict_core==20260723` | | `zh` | `jieba==0.42.1` | | `ar` | `snowballstemmer==3.1.1` | @@ -24,3 +27,17 @@ Package versions are pinned by Discrawl: Installation is always explicit. Opening an archive or running a search never downloads or installs code. Tokenizer workers start lazily only when an enabled language is first used for indexing or search. + +For Korean, configure: + +```toml +[search.lexical] +languages = ["ko"] +kiwi_command = "~/.local/share/discrawl/bin/discrawl-kiwi" +kiwi_model = "~/.local/share/discrawl/models/kiwi/base" +``` + +The helper source is under `tools/discrawl-kiwi`. It uses the existing +[`github.com/codingpot/kiwigo`](https://pkg.go.dev/github.com/codingpot/kiwigo) +binding and dynamically linked Kiwi library; it does not use Python or +`kiwipiepy`. diff --git a/docs/configuration.md b/docs/configuration.md index e23ea12b..80beb358 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -80,6 +80,8 @@ default_mode = "fts" [search.lexical] languages = [] # optional: "ko", "ja", "zh", "ar" python = "python3" +kiwi_command = "discrawl-kiwi" +kiwi_model = "" [search.embeddings] enabled = false @@ -144,9 +146,10 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and - `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion - `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement` - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time -- `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi), Japanese (`ja`, Sudachi), Chinese (`zh`, Jieba), and Arabic (`ar`, Snowball plus proclitic splitting). -- `discrawl lexical install` installs only the pinned packages required by the configured languages and refuses system-Python installation. Search and sync never install packages implicitly. -- Tokenizer workers load lazily on first indexing or search use. Commands that only inspect metadata do not start Python or require tokenizer modules. +- `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi through the `github.com/codingpot/kiwigo` Go binding), Japanese (`ja`, Sudachi), Chinese (`zh`, Jieba), and Arabic (`ar`, Snowball plus proclitic splitting). +- Korean requires the separately built `discrawl-kiwi` helper plus Kiwi 0.23.2's dynamic library and base model. Set `kiwi_command` and `kiwi_model` to their installed paths. Korean does not use Python or `kiwipiepy`. +- `discrawl lexical install` installs only the pinned Python packages required by configured Japanese, Chinese, or Arabic fields and refuses system-Python installation. Search and sync never install packages implicitly. +- Tokenizer workers load lazily on first indexing or search use. Commands that only inspect metadata do not start the Kiwi helper or Python modules. - Each enabled language adds an independent FTS5 table. Index and query text pass through the same tokenizer, and results from the default plus language-specific tables are merged with reciprocal rank fusion. - After adding or changing `search.lexical.languages`, run a writer command such as `discrawl sync` once so the configured lexical tables are built. Read-only commands never mutate the archive; new and edited messages update the tables automatically during later syncs. - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search diff --git a/docs/guides/search-modes.md b/docs/guides/search-modes.md index 9ffd323e..eb844edc 100644 --- a/docs/guides/search-modes.md +++ b/docs/guides/search-modes.md @@ -12,28 +12,43 @@ - backed by SQLite FTS5 with the default `unicode61` tokenizer - optional `[search.lexical]` languages add independent tokenizer-specific FTS tables and merge their ranked results with reciprocal rank fusion -- supported presets are Korean with Kiwi, Japanese with Sudachi A-mode, Chinese with Jieba search mode, and Arabic with Snowball stemming plus proclitic splitting +- supported presets are Korean with the native Kiwi engine through the `kiwigo` Go binding, Japanese with Sudachi A-mode, Chinese with Jieba search mode, and Arabic with Snowball stemming plus proclitic splitting - user query terms are parameterized and quoted before `MATCH`, so tokens like `AND`, `OR`, `NOT`, `NEAR`, and `*` are searched as input terms instead of FTS operators - punctuation still follows FTS5 tokenization rules - by default, `search` skips rows with no searchable content (attachment text, attachment filenames, embeds, and replies still count as content); use `--include-empty` to opt back in ### Optional multilingual lexical fields -Create an isolated Python environment: +Install Kiwi 0.23.2's dynamic library and base model, then build the Go helper: + +```bash +git clone https://github.com/openclaw/discrawl +cd discrawl/tools/discrawl-kiwi +go build -o ~/.local/share/discrawl/bin/discrawl-kiwi . +``` + +`github.com/codingpot/kiwigo` links to the system Kiwi C API. Its upstream +installation expects Kiwi headers and dynamic libraries under `/usr/local`; +the model is the `kiwi_model_v0.23.2_base.tgz` release asset. + +Create an isolated Python environment only if Japanese, Chinese, or Arabic is +enabled: ```bash python3 -m venv ~/.local/share/discrawl/tokenizers ``` -Then configure the fields: +Configure the fields: ```toml [search.lexical] languages = ["ko", "ja", "zh", "ar"] python = "~/.local/share/discrawl/tokenizers/bin/python" # ~ is expanded +kiwi_command = "~/.local/share/discrawl/bin/discrawl-kiwi" +kiwi_model = "~/.local/share/discrawl/models/kiwi/base" ``` -Install only the packages selected by `languages`: +Install only the Python packages selected by non-Korean languages: ```bash discrawl lexical install @@ -45,9 +60,11 @@ through every enabled analyzer. Disk usage and indexing work increase with the number of fields; query-time RRF deduplicates message ids without mixing the different BM25 term statistics into one field. -Discrawl never installs packages during archive open, sync, or search. Workers -are loaded lazily on first use, while `lexical install` is an explicit, -virtual-environment-only network operation. +Korean text never crosses a Python boundary: `discrawl-kiwi` is a persistent +Go helper built against `github.com/codingpot/kiwigo` and dynamically linked to +Kiwi. Discrawl never installs packages during archive open, sync, or search. +All helpers are loaded lazily on first use, while `lexical install` is an +explicit, virtual-environment-only operation for the remaining analyzers. See [Multilingual lexical benchmark](../benchmarks/multilingual-lexical.html) for the reproducible targeted quality check and its storage tradeoff. diff --git a/internal/cli/cli.go b/internal/cli/cli.go index e9dcd8fb..24f83173 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -128,7 +128,7 @@ var discrawlCommandSpecs = []discrawlCommandSpec{ {name: "cache-import", description: "Import Discord Desktop cache data (wiretap alias)."}, {name: "wiretap", description: "Import Discord Desktop cache data."}, {name: "search", description: "Search archived messages."}, - {name: "lexical", description: "Install configured multilingual lexical tokenizers."}, + {name: "lexical", description: "Install configured Python lexical tokenizers."}, {name: "tui", description: "Explore the archive in an interactive terminal UI."}, {name: "messages", description: "List archived messages."}, {name: "digest", description: "Summarize recent archive activity."}, @@ -694,8 +694,10 @@ func (r *runtime) openConfiguredReadOnlyStore(path string) (*store.Store, error) return store.OpenReadOnly(r.ctx, path) } return store.OpenReadOnlyWithOptions(r.ctx, path, store.OpenOptions{ - LexicalLanguages: r.cfg.Search.Lexical.Languages, - LexicalPython: r.cfg.Search.Lexical.Python, + LexicalLanguages: r.cfg.Search.Lexical.Languages, + LexicalPython: r.cfg.Search.Lexical.Python, + LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand, + LexicalKiwiModel: r.cfg.Search.Lexical.KiwiModel, }) } @@ -771,8 +773,10 @@ func (r *runtime) localStoreFactory() func(context.Context, string) (*store.Stor } return func(ctx context.Context, path string) (*store.Store, error) { return store.OpenWithOptions(ctx, path, store.OpenOptions{ - LexicalLanguages: r.cfg.Search.Lexical.Languages, - LexicalPython: r.cfg.Search.Lexical.Python, + LexicalLanguages: r.cfg.Search.Lexical.Languages, + LexicalPython: r.cfg.Search.Lexical.Python, + LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand, + LexicalKiwiModel: r.cfg.Search.Lexical.KiwiModel, }) } } diff --git a/internal/cli/lexical_commands.go b/internal/cli/lexical_commands.go index 794be976..e4b3954c 100644 --- a/internal/cli/lexical_commands.go +++ b/internal/cli/lexical_commands.go @@ -11,6 +11,7 @@ type lexicalInstallOutput struct { Languages []string `json:"languages"` Packages []string `json:"packages"` Python string `json:"python"` + Kiwi string `json:"kiwi,omitempty"` } func (r *runtime) runLexical(args []string) error { @@ -34,7 +35,8 @@ func (r *runtime) runLexical(args []string) error { } return r.print(lexicalInstallOutput{ Languages: append([]string(nil), r.cfg.Search.Lexical.Languages...), - Packages: result.Packages, + Packages: append([]string{}, result.Packages...), Python: r.cfg.Search.Lexical.Python, + Kiwi: r.cfg.Search.Lexical.KiwiCommand, }) } diff --git a/internal/cli/lexical_commands_test.go b/internal/cli/lexical_commands_test.go index 02e82c25..09c76c7a 100644 --- a/internal/cli/lexical_commands_test.go +++ b/internal/cli/lexical_commands_test.go @@ -30,12 +30,11 @@ func TestRunLexicalInstallUsesConfiguredLanguages(t *testing.T) { require.Equal(t, "/tmp/tokenizers/bin/python", python) require.Equal(t, []string{"ko", "zh"}, languages) return store.LexicalInstallResult{ - Packages: []string{"kiwipiepy==0.23.2", "jieba==0.42.1"}, + Packages: []string{"jieba==0.42.1"}, }, nil } require.NoError(t, r.runLexical([]string{"install"})) - require.Contains(t, stdout.String(), "kiwipiepy==0.23.2") require.Contains(t, stdout.String(), "jieba==0.42.1") } @@ -76,7 +75,7 @@ func TestRunLexicalInstallJSONOutput(t *testing.T) { stdout: &stdout, json: true, installLexical: func(context.Context, string, []string) (store.LexicalInstallResult, error) { - return store.LexicalInstallResult{Packages: []string{"kiwipiepy==0.23.2"}}, nil + return store.LexicalInstallResult{}, nil }, } r.cfg.Search.Lexical.Languages = []string{"ko"} @@ -84,8 +83,9 @@ func TestRunLexicalInstallJSONOutput(t *testing.T) { require.NoError(t, r.runLexical([]string{"install"})) require.JSONEq(t, `{ "languages": ["ko"], - "packages": ["kiwipiepy==0.23.2"], - "python": "python3" + "packages": [], + "python": "python3", + "kiwi": "discrawl-kiwi" }`, stdout.String()) } diff --git a/internal/cli/output.go b/internal/cli/output.go index 610f0710..3d2cc6d5 100644 --- a/internal/cli/output.go +++ b/internal/cli/output.go @@ -252,8 +252,8 @@ Flags: "lexical": `Usage: discrawl lexical install -Install pinned tokenizer packages for the languages in search.lexical.languages. -The configured Python interpreter must belong to a virtual environment. +Install pinned Python packages for configured Japanese, Chinese, and Arabic fields. +Korean uses the separately built discrawl-kiwi Go helper and requires no Python package. `, "attachments": `Usage: discrawl attachments [flags] diff --git a/internal/config/config.go b/internal/config/config.go index 95390991..a490b4bc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -73,8 +73,10 @@ type SearchConfig struct { } type LexicalSearchConfig struct { - Languages []string `toml:"languages,omitempty"` - Python string `toml:"python"` + Languages []string `toml:"languages,omitempty"` + Python string `toml:"python"` + KiwiCommand string `toml:"kiwi_command"` + KiwiModel string `toml:"kiwi_model"` } type ShareConfig struct { @@ -161,7 +163,8 @@ func Default() Config { Search: SearchConfig{ DefaultMode: "fts", Lexical: LexicalSearchConfig{ - Python: "python3", + Python: "python3", + KiwiCommand: "discrawl-kiwi", }, Embeddings: EmbeddingsConfig{ Enabled: false, @@ -313,6 +316,11 @@ func (c *Config) Normalize() error { if c.Search.Lexical.Python == "" { c.Search.Lexical.Python = "python3" } + c.Search.Lexical.KiwiCommand = strings.TrimSpace(c.Search.Lexical.KiwiCommand) + if c.Search.Lexical.KiwiCommand == "" { + c.Search.Lexical.KiwiCommand = "discrawl-kiwi" + } + c.Search.Lexical.KiwiModel = strings.TrimSpace(c.Search.Lexical.KiwiModel) seenLexicalLanguages := make(map[string]struct{}, len(c.Search.Lexical.Languages)) normalizedLexicalLanguages := make([]string, 0, len(c.Search.Lexical.Languages)) for _, language := range c.Search.Lexical.Languages { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4562ca83..5544b2fa 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -262,12 +262,16 @@ token_source = "env" [search.lexical] languages = ["ko", "ja", "zh", "ar"] python = "/opt/discrawl-tokenizers/bin/python" +kiwi_command = "/opt/discrawl/bin/discrawl-kiwi" +kiwi_model = "/opt/discrawl/models/kiwi/base" `), 0o600)) cfg, err := Load(path) require.NoError(t, err) require.Equal(t, []string{"ko", "ja", "zh", "ar"}, cfg.Search.Lexical.Languages) require.Equal(t, "/opt/discrawl-tokenizers/bin/python", cfg.Search.Lexical.Python) + require.Equal(t, "/opt/discrawl/bin/discrawl-kiwi", cfg.Search.Lexical.KiwiCommand) + require.Equal(t, "/opt/discrawl/models/kiwi/base", cfg.Search.Lexical.KiwiModel) } func TestNormalizeRejectsUnsupportedLexicalLanguage(t *testing.T) { diff --git a/internal/store/lexical_install.go b/internal/store/lexical_install.go index 990dd4d9..fbaf375a 100644 --- a/internal/store/lexical_install.go +++ b/internal/store/lexical_install.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "os" "os/exec" "runtime" "strings" @@ -36,7 +37,10 @@ func installLexicalPackagesWithRunner( return LexicalInstallResult{}, err } if len(packages) == 0 { - return LexicalInstallResult{}, errors.New("no search.lexical languages are configured") + if len(languages) == 0 { + return LexicalInstallResult{}, errors.New("no search.lexical languages are configured") + } + return LexicalInstallResult{}, nil } python, err = expandLexicalPython(python) if err != nil { @@ -88,7 +92,7 @@ func lexicalPythonPackages(languages []string) ([]string, error) { seen[language] = struct{}{} switch language { case "ko": - packages = append(packages, "kiwipiepy==0.23.2") + continue case "ja": packages = append(packages, "sudachipy==0.6.11", "sudachidict_core==20260723") case "zh": @@ -107,6 +111,7 @@ func runLexicalCommand(ctx context.Context, path string, args ...string) ([]byte cmd := &exec.Cmd{ Path: path, Args: append([]string{path}, args...), + Env: lexicalWorkerEnvironment(os.Environ()), Stdout: &output, Stderr: &output, } diff --git a/internal/store/lexical_kiwi.go b/internal/store/lexical_kiwi.go new file mode 100644 index 00000000..6f0cb1a4 --- /dev/null +++ b/internal/store/lexical_kiwi.go @@ -0,0 +1,211 @@ +package store + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type kiwiLexicalResponse struct { + Ready bool `json:"ready,omitempty"` + Tokens string `json:"tokens,omitempty"` + Error string `json:"error,omitempty"` +} + +func startKiwiLexicalTokenizer(command, model string) (LexicalTokenizer, error) { + cmd, err := newKiwiLexicalCommand(command, model) + if err != nil { + return nil, err + } + return startKiwiLexicalTokenizerCommand(cmd) +} + +func startKiwiLexicalTokenizerCommand(cmd *exec.Cmd) (LexicalTokenizer, error) { + tokenizer := &externalLexicalTokenizer{ + language: "ko", + command: cmd, + } + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + _ = stdin.Close() + return nil, err + } + cmd.Stderr = &tokenizer.stderr + tokenizer.stdin = stdin + tokenizer.stdout = bufio.NewScanner(stdout) + tokenizer.stdout.Buffer(make([]byte, 4096), 8*1024*1024) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start Korean Kiwi tokenizer: %w", err) + } + response, err := tokenizer.readStartup() + if err != nil { + _ = tokenizer.Close() + return nil, fmt.Errorf("initialize Korean Kiwi tokenizer: %w", err) + } + if !response.Ready { + _ = tokenizer.Close() + return nil, errors.New(response.Error) + } + return tokenizer, nil +} + +func newKiwiLexicalCommand(command, model string) (*exec.Cmd, error) { + command, err := expandLexicalPath(command, "discrawl-kiwi") + if err != nil { + return nil, fmt.Errorf("resolve Kiwi helper: %w", err) + } + if !filepath.IsAbs(command) { + if command != "discrawl-kiwi" { + return nil, fmt.Errorf( + "unsupported Kiwi helper %q; use an absolute path or discrawl-kiwi", + command, + ) + } + command, err = exec.LookPath(command) + if err != nil { + return nil, fmt.Errorf("find Kiwi helper: %w", err) + } + } + model, err = expandLexicalPath(model, "") + if err != nil { + return nil, fmt.Errorf("resolve Kiwi model: %w", err) + } + args := []string{command} + if model != "" { + args = append(args, "--model", model) + } + return &exec.Cmd{ + Path: command, + Args: args, + Env: lexicalWorkerEnvironment(os.Environ()), + }, nil +} + +func expandLexicalPath(path, fallback string) (string, error) { + path = strings.TrimSpace(path) + if path == "" { + return fallback, nil + } + if !strings.HasPrefix(path, "~/") { + return path, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, strings.TrimPrefix(path, "~/")), nil +} + +type externalLexicalTokenizer struct { + language string + command *exec.Cmd + stdin io.WriteCloser + stdout *bufio.Scanner + stderr lockedBuffer + mutex sync.Mutex +} + +func (t *externalLexicalTokenizer) readStartup() (kiwiLexicalResponse, error) { + result := make(chan struct { + response kiwiLexicalResponse + err error + }, 1) + go func() { + response, err := t.readResponse() + result <- struct { + response kiwiLexicalResponse + err error + }{response, err} + }() + select { + case output := <-result: + return output.response, output.err + case <-time.After(30 * time.Second): + _ = t.command.Process.Kill() + return kiwiLexicalResponse{}, errors.New("kiwi tokenizer startup timed out after 30s") + } +} + +func (t *externalLexicalTokenizer) Tokenize(ctx context.Context, text string) (string, error) { + t.mutex.Lock() + defer t.mutex.Unlock() + if err := ctx.Err(); err != nil { + return "", err + } + request, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return "", err + } + if _, err := t.stdin.Write(append(request, '\n')); err != nil { + return "", fmt.Errorf("write %s tokenizer request: %w", t.language, err) + } + result := make(chan struct { + response kiwiLexicalResponse + err error + }, 1) + go func() { + response, err := t.readResponse() + result <- struct { + response kiwiLexicalResponse + err error + }{response, err} + }() + select { + case output := <-result: + if output.err != nil { + return "", output.err + } + if output.response.Error != "" { + return "", errors.New(output.response.Error) + } + return output.response.Tokens, nil + case <-ctx.Done(): + _ = t.command.Process.Kill() + return "", ctx.Err() + case <-time.After(30 * time.Second): + _ = t.command.Process.Kill() + return "", errors.New("kiwi tokenizer response timed out after 30s") + } +} + +func (t *externalLexicalTokenizer) readResponse() (kiwiLexicalResponse, error) { + if !t.stdout.Scan() { + if err := t.stdout.Err(); err != nil { + return kiwiLexicalResponse{}, err + } + if detail := strings.TrimSpace(t.stderr.String()); detail != "" { + return kiwiLexicalResponse{}, errors.New(detail) + } + return kiwiLexicalResponse{}, io.EOF + } + var response kiwiLexicalResponse + if err := json.Unmarshal(t.stdout.Bytes(), &response); err != nil { + return kiwiLexicalResponse{}, fmt.Errorf("decode Kiwi tokenizer response: %w", err) + } + return response, nil +} + +func (t *externalLexicalTokenizer) Close() error { + if t == nil || t.command == nil || t.command.Process == nil { + return nil + } + _ = t.stdin.Close() + err := t.command.Wait() + if errors.Is(err, os.ErrProcessDone) { + return nil + } + return err +} diff --git a/internal/store/lexical_python.go b/internal/store/lexical_python.go index b603d9be..8ebae1af 100644 --- a/internal/store/lexical_python.go +++ b/internal/store/lexical_python.go @@ -17,8 +17,10 @@ import ( ) type OpenOptions struct { - LexicalLanguages []string - LexicalPython string + LexicalLanguages []string + LexicalPython string + LexicalKiwiCommand string + LexicalKiwiModel string } type pythonLexicalTokenizer struct { @@ -54,7 +56,7 @@ type pythonLexicalResponse struct { } func OpenWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { - tokenizers, err := newPythonLexicalTokenizers(opts) + tokenizers, err := newLexicalTokenizers(opts) if err != nil { return nil, err } @@ -62,14 +64,14 @@ func OpenWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store } func OpenReadOnlyWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { - tokenizers, err := newPythonLexicalTokenizers(opts) + tokenizers, err := newLexicalTokenizers(opts) if err != nil { return nil, err } return openReadOnlyWithLexicalTokenizers(ctx, path, tokenizers) } -func newPythonLexicalTokenizers(opts OpenOptions) (map[string]LexicalTokenizer, error) { +func newLexicalTokenizers(opts OpenOptions) (map[string]LexicalTokenizer, error) { if len(opts.LexicalLanguages) == 0 { return nil, nil } @@ -79,9 +81,22 @@ func newPythonLexicalTokenizers(opts OpenOptions) (map[string]LexicalTokenizer, } tokenizers := make(map[string]LexicalTokenizer, len(opts.LexicalLanguages)) for _, language := range opts.LexicalLanguages { - tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { - return startPythonLexicalTokenizer(python, language) - }) + switch language { + case "ko": + command := opts.LexicalKiwiCommand + model := opts.LexicalKiwiModel + tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + tokenizer, err := startKiwiLexicalTokenizer(command, model) + if err != nil { + return nil, fmt.Errorf("start ko lexical tokenizer: %w", err) + } + return tokenizer, nil + }) + default: + tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + return startPythonLexicalTokenizer(python, language) + }) + } } return tokenizers, nil } @@ -250,9 +265,6 @@ language = sys.argv[1] try: if language == "default": engine = None - elif language == "ko": - from kiwipiepy import Kiwi - engine = Kiwi() elif language == "ja": from sudachipy import dictionary, tokenizer as sudachi_tokenizer engine = dictionary.Dictionary().create() @@ -284,8 +296,6 @@ def unique(tokens): def tokenize(text): if language == "default": return unique(re.findall(r"[^\W\d_]+", text, flags=re.UNICODE)) - if language == "ko": - return unique(token.form for token in engine.tokenize(text) if not token.tag.startswith("S")) if language == "ja": output = [] for token in engine.tokenize(text, split_mode): diff --git a/internal/store/lexical_python_test.go b/internal/store/lexical_python_test.go index dad68d5a..895d7785 100644 --- a/internal/store/lexical_python_test.go +++ b/internal/store/lexical_python_test.go @@ -100,8 +100,8 @@ func TestPythonLexicalTokenizerReportsWriteAfterClose(t *testing.T) { require.ErrorContains(t, err, "write tokenizer request") } -func TestNewPythonLexicalTokenizersDisabled(t *testing.T) { - tokenizers, err := newPythonLexicalTokenizers(OpenOptions{}) +func TestNewLexicalTokenizersDisabled(t *testing.T) { + tokenizers, err := newLexicalTokenizers(OpenOptions{}) require.NoError(t, err) require.Nil(t, tokenizers) } @@ -116,14 +116,14 @@ func TestPythonLexicalTokenizerDefaultWorker(t *testing.T) { require.Equal(t, "mixed case", tokens) } -func TestNewPythonLexicalTokenizersExpandsHomePath(t *testing.T) { +func TestNewLexicalTokenizersExpandsPythonHomePath(t *testing.T) { python, err := exec.LookPath("python3") require.NoError(t, err) home := t.TempDir() require.NoError(t, os.Symlink(python, filepath.Join(home, "python3"))) t.Setenv("HOME", home) - tokenizers, err := newPythonLexicalTokenizers(OpenOptions{ + tokenizers, err := newLexicalTokenizers(OpenOptions{ LexicalLanguages: []string{"default"}, LexicalPython: "~/python3", }) @@ -132,11 +132,12 @@ func TestNewPythonLexicalTokenizersExpandsHomePath(t *testing.T) { closeLexicalTokenizers(tokenizers) } -func TestOpenWithOptionsLoadsPythonLazily(t *testing.T) { +func TestOpenWithOptionsLoadsKiwiHelperLazily(t *testing.T) { ctx := context.Background() s, err := OpenWithOptions(ctx, filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ - LexicalLanguages: []string{"ko"}, - LexicalPython: "/definitely/missing/discrawl-python", + LexicalLanguages: []string{"ko"}, + LexicalKiwiCommand: "/definitely/missing/discrawl-kiwi", + LexicalKiwiModel: "/definitely/missing/kiwi-model", }) require.NoError(t, err) defer func() { _ = s.Close() }() @@ -147,6 +148,7 @@ func TestOpenWithOptionsLoadsPythonLazily(t *testing.T) { Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, }) require.ErrorContains(t, err, "start ko lexical tokenizer") + require.NotContains(t, err.Error(), "Python") } func lexicalHelperCommand(mode string) *exec.Cmd { @@ -186,8 +188,17 @@ func TestLexicalTokenizerHelperProcess(t *testing.T) { fmt.Println(`{"error":"tokenization failed"}`) continue } + if mode == "malformed-response" { + fmt.Println(`not-json`) + continue + } response, err := json.Marshal(map[string]string{ - "tokens": request["text"] + " tokenized", + "tokens": func() string { + if request["text"] == "오늘 저녁먹음 기록" { + return "오늘 저녁 먹 음 기록" + } + return request["text"] + " tokenized" + }(), }) if err != nil { fmt.Printf("{\"error\":%q}\n", err.Error()) diff --git a/internal/store/lexical_review_test.go b/internal/store/lexical_review_test.go index d7b72523..e4906565 100644 --- a/internal/store/lexical_review_test.go +++ b/internal/store/lexical_review_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "slices" "strings" "testing" @@ -16,9 +17,9 @@ func TestLexicalPythonPackagesSelectOnlyConfiguredLanguages(t *testing.T) { packages, err := lexicalPythonPackages([]string{"ko", "zh"}) require.NoError(t, err) require.Equal(t, []string{ - "kiwipiepy==0.23.2", "jieba==0.42.1", }, packages) + require.NotContains(t, packages, "kiwipiepy==0.23.2") require.NotContains(t, packages, "sudachipy==0.6.11") require.NotContains(t, packages, "snowballstemmer==3.1.1") } @@ -37,11 +38,17 @@ func TestInstallLexicalPackagesRejectsNoConfiguredLanguages(t *testing.T) { require.ErrorContains(t, err, "no search.lexical languages") } +func TestInstallLexicalPackagesSkipsKiwiOnlyConfiguration(t *testing.T) { + result, err := InstallLexicalPackages(context.Background(), "python3", []string{"ko"}) + require.NoError(t, err) + require.Empty(t, result.Packages) +} + func TestInstallLexicalPackagesRequiresVirtualEnvironment(t *testing.T) { _, err := installLexicalPackagesWithRunner( context.Background(), "/tmp/python", - []string{"ko"}, + []string{"zh"}, func(context.Context, string, ...string) ([]byte, error) { return []byte("false\n"), nil }, @@ -98,7 +105,7 @@ func TestInstallLexicalPackagesReportsBoundaryFailures(t *testing.T) { _, err = installLexicalPackagesWithRunner( context.Background(), "sh", - []string{"ko"}, + []string{"zh"}, func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("must not run") }, @@ -108,7 +115,7 @@ func TestInstallLexicalPackagesReportsBoundaryFailures(t *testing.T) { _, err = installLexicalPackagesWithRunner( context.Background(), "/tmp/python", - []string{"ko"}, + []string{"zh"}, func(context.Context, string, ...string) ([]byte, error) { return nil, errors.New("probe failed") }, @@ -119,7 +126,7 @@ func TestInstallLexicalPackagesReportsBoundaryFailures(t *testing.T) { _, err = installLexicalPackagesWithRunner( context.Background(), "/tmp/python", - []string{"ko"}, + []string{"zh"}, func(context.Context, string, ...string) ([]byte, error) { calls++ if calls == 1 { @@ -175,6 +182,103 @@ func TestLexicalWorkerEnvironmentDropsParentSecrets(t *testing.T) { require.NotContains(t, joined, "password") } +func TestKiwiCommandUsesGoHelperAndConfiguredModel(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + command, err := newKiwiLexicalCommand( + "/opt/discrawl/bin/discrawl-kiwi", + "~/models/kiwi/base", + ) + require.NoError(t, err) + require.Equal(t, "/opt/discrawl/bin/discrawl-kiwi", command.Path) + require.Equal(t, []string{ + "/opt/discrawl/bin/discrawl-kiwi", + "--model", + filepath.Join(home, "models/kiwi/base"), + }, command.Args) +} + +func TestKiwiCommandRejectsArbitraryRelativeCommand(t *testing.T) { + _, err := newKiwiLexicalCommand("sh", "/tmp/model") + require.ErrorContains(t, err, "unsupported Kiwi helper") +} + +func TestKiwiTokenizerCommandProtocol(t *testing.T) { + tokenizer, err := startKiwiLexicalTokenizerCommand( + lexicalHelperCommand("ready"), + ) + require.NoError(t, err) + + tokens, err := tokenizer.Tokenize(context.Background(), "오늘 저녁먹음 기록") + require.NoError(t, err) + require.Equal(t, "오늘 저녁 먹 음 기록", tokens) + require.NoError(t, tokenizer.Close()) + _, err = tokenizer.Tokenize(context.Background(), "text") + require.ErrorContains(t, err, "write ko tokenizer request") +} + +func TestKiwiTokenizerCommandStartupFailures(t *testing.T) { + tokenizer, err := startKiwiLexicalTokenizerCommand( + lexicalHelperCommand("startup-error"), + ) + require.Nil(t, tokenizer) + require.ErrorContains(t, err, "missing tokenizer package") + + tokenizer, err = startKiwiLexicalTokenizerCommand( + lexicalHelperCommand("malformed-startup"), + ) + require.Nil(t, tokenizer) + require.ErrorContains(t, err, "decode Kiwi tokenizer response") + + tokenizer, err = startKiwiLexicalTokenizerCommand( + lexicalHelperCommand("stderr-startup"), + ) + require.Nil(t, tokenizer) + require.ErrorContains(t, err, "tokenizer stderr") +} + +func TestKiwiTokenizerCommandResponseErrorAndCancellation(t *testing.T) { + tokenizer, err := startKiwiLexicalTokenizerCommand( + lexicalHelperCommand("response-error"), + ) + require.NoError(t, err) + defer func() { _ = tokenizer.Close() }() + + _, err = tokenizer.Tokenize(context.Background(), "text") + require.ErrorContains(t, err, "tokenization failed") + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = tokenizer.Tokenize(ctx, "text") + require.ErrorIs(t, err, context.Canceled) +} + +func TestKiwiTokenizerCommandMalformedResponse(t *testing.T) { + tokenizer, err := startKiwiLexicalTokenizerCommand( + lexicalHelperCommand("malformed-response"), + ) + require.NoError(t, err) + defer func() { _ = tokenizer.Close() }() + + _, err = tokenizer.Tokenize(context.Background(), "text") + require.ErrorContains(t, err, "decode Kiwi tokenizer response") + + var nilTokenizer *externalLexicalTokenizer + require.NoError(t, nilTokenizer.Close()) +} + +func TestKiwiCommandDefaultHelperAndOptionalModel(t *testing.T) { + bin := t.TempDir() + helper := filepath.Join(bin, "discrawl-kiwi") + require.NoError(t, os.WriteFile(helper, []byte("#!/bin/sh\n"), 0o700)) + t.Setenv("PATH", bin) + + command, err := newKiwiLexicalCommand("", "") + require.NoError(t, err) + require.Equal(t, helper, command.Path) + require.Equal(t, []string{helper}, command.Args) +} + func TestRunLexicalCommandCapturesOutputAndFailure(t *testing.T) { t.Setenv("DISCRAWL_INSTALL_HELPER", "1") output, err := runLexicalCommand( @@ -185,7 +289,7 @@ func TestRunLexicalCommandCapturesOutputAndFailure(t *testing.T) { "success", ) require.NoError(t, err) - require.Equal(t, "installed\n", string(output)) + require.Contains(t, string(output), "installed\n") output, err = runLexicalCommand( context.Background(), @@ -201,8 +305,23 @@ func TestRunLexicalCommandCapturesOutputAndFailure(t *testing.T) { require.Error(t, err) } +func TestRunLexicalCommandDropsParentSecrets(t *testing.T) { + t.Setenv("DISCORD_BOT_TOKEN", "discord-secret") + t.Setenv("OPENAI_API_KEY", "openai-secret") + t.Setenv("PIP_INDEX_URL", "https://user:password@example.invalid/simple") + output, err := runLexicalCommand( + context.Background(), + os.Args[0], + "-test.run=TestLexicalInstallCommandHelperProcess", + "--", + "environment", + ) + require.NoError(t, err) + require.Contains(t, string(output), "clean\n") +} + func TestLexicalInstallCommandHelperProcess(t *testing.T) { - if os.Getenv("DISCRAWL_INSTALL_HELPER") != "1" { + if !slices.Contains(os.Args, "-test.run=TestLexicalInstallCommandHelperProcess") { return } switch os.Args[len(os.Args)-1] { @@ -212,6 +331,15 @@ func TestLexicalInstallCommandHelperProcess(t *testing.T) { case "failure": fmt.Println("install failed") os.Exit(2) + case "environment": + for _, key := range []string{"DISCORD_BOT_TOKEN", "OPENAI_API_KEY", "PIP_INDEX_URL"} { + if os.Getenv(key) != "" { + fmt.Println(key) + os.Exit(4) + } + } + fmt.Println("clean") + os.Exit(0) default: os.Exit(3) } diff --git a/internal/store/multilingual_benchmark_test.go b/internal/store/multilingual_benchmark_test.go index 60ac3ae0..1fb6173d 100644 --- a/internal/store/multilingual_benchmark_test.go +++ b/internal/store/multilingual_benchmark_test.go @@ -27,8 +27,10 @@ func TestMultilingualLexicalQualityBenchmark(t *testing.T) { require.NoError(t, err) defer func() { _ = baseline.Close() }() multilingual, err := OpenWithOptions(ctx, filepath.Join(root, "multilingual.db"), OpenOptions{ - LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, - LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), + LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, + LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), + LexicalKiwiCommand: os.Getenv("DISCRAWL_KIWI_HELPER"), + LexicalKiwiModel: os.Getenv("DISCRAWL_KIWI_MODEL"), }) require.NoError(t, err) defer func() { _ = multilingual.Close() }() diff --git a/internal/store/multilingual_python_e2e_test.go b/internal/store/multilingual_e2e_test.go similarity index 84% rename from internal/store/multilingual_python_e2e_test.go rename to internal/store/multilingual_e2e_test.go index 0589ae88..9a270b40 100644 --- a/internal/store/multilingual_python_e2e_test.go +++ b/internal/store/multilingual_e2e_test.go @@ -10,14 +10,16 @@ import ( "github.com/stretchr/testify/require" ) -func TestPythonMultilingualLexicalSearchE2E(t *testing.T) { +func TestMultilingualLexicalSearchE2E(t *testing.T) { if os.Getenv("DISCRAWL_TOKENIZER_E2E") != "1" { t.Skip("set DISCRAWL_TOKENIZER_E2E=1 with optional tokenizer packages installed") } ctx := context.Background() s, err := OpenWithOptions(ctx, filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ - LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, - LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), + LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, + LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), + LexicalKiwiCommand: os.Getenv("DISCRAWL_KIWI_HELPER"), + LexicalKiwiModel: os.Getenv("DISCRAWL_KIWI_MODEL"), }) require.NoError(t, err) defer func() { _ = s.Close() }() diff --git a/internal/store/multilingual_lifecycle_test.go b/internal/store/multilingual_lifecycle_test.go index 03e7b50d..d0861bc3 100644 --- a/internal/store/multilingual_lifecycle_test.go +++ b/internal/store/multilingual_lifecycle_test.go @@ -182,8 +182,9 @@ func TestOpenReadOnlyWithOptionsKeepsMissingTokenizerLazy(t *testing.T) { require.NoError(t, writer.Close()) reader, err := OpenReadOnlyWithOptions(ctx, path, OpenOptions{ - LexicalLanguages: []string{"ko"}, - LexicalPython: "/definitely/missing/discrawl-python", + LexicalLanguages: []string{"ko"}, + LexicalKiwiCommand: "/definitely/missing/discrawl-kiwi", + LexicalKiwiModel: "/definitely/missing/kiwi-model", }) require.NoError(t, err) require.NoError(t, reader.Close()) diff --git a/tools/discrawl-kiwi/README.md b/tools/discrawl-kiwi/README.md new file mode 100644 index 00000000..465d1a28 --- /dev/null +++ b/tools/discrawl-kiwi/README.md @@ -0,0 +1,47 @@ +# discrawl-kiwi + +`discrawl-kiwi` is Discrawl's optional Korean lexical analyzer helper. It uses +the existing [`github.com/codingpot/kiwigo`](https://pkg.go.dev/github.com/codingpot/kiwigo) +Go binding and Kiwi's public C API. It does not use Python. + +## Native prerequisites + +- Kiwi 0.23.2 headers and dynamic library +- Kiwi 0.23.2 base model +- a C/C++ toolchain supported by CGO + +The `kiwigo` build currently looks for headers and libraries under +`/usr/local/include` and `/usr/local/lib`. The official Kiwi release assets are: + +- `kiwi___v0.23.2.tgz` +- `kiwi_model_v0.23.2_base.tgz` + +Build: + +```bash +bash install-kiwi.sh +go build -o discrawl-kiwi . +``` + +Run: + +```bash +discrawl-kiwi --model /path/to/models/cong/base +``` + +The helper speaks newline-delimited JSON over stdin/stdout and stays alive so +the model is loaded once: + +```text +{"ready":true,"version":"0.23.2"} +{"text":"오늘 저녁먹음 기록"} +{"tokens":"오늘 저녁 먹 음 기록"} +``` + +## License boundary + +Kiwi and `github.com/codingpot/kiwigo` are licensed under +LGPL-2.1-or-later. Discrawl invokes this separately distributed helper as an +optional process, and the helper dynamically links to the replaceable Kiwi +library. Distributors of the helper or Kiwi binary assets must include the +applicable LGPL notices and corresponding Kiwi source access. diff --git a/tools/discrawl-kiwi/go.mod b/tools/discrawl-kiwi/go.mod new file mode 100644 index 00000000..54853644 --- /dev/null +++ b/tools/discrawl-kiwi/go.mod @@ -0,0 +1,14 @@ +module github.com/openclaw/discrawl/tools/discrawl-kiwi + +go 1.26 + +require ( + github.com/codingpot/kiwigo v0.0.0-20260812004023-ba165a3d4d4e + github.com/stretchr/testify v1.7.0 +) + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect +) diff --git a/tools/discrawl-kiwi/go.sum b/tools/discrawl-kiwi/go.sum new file mode 100644 index 00000000..145ac1a7 --- /dev/null +++ b/tools/discrawl-kiwi/go.sum @@ -0,0 +1,15 @@ +github.com/codingpot/kiwigo v0.0.0-20260812004023-ba165a3d4d4e h1:nAYTYAwQxCBScKgc5uWTTI2MJym/5i+zImK57kL0e8I= +github.com/codingpot/kiwigo v0.0.0-20260812004023-ba165a3d4d4e/go.mod h1:dBnWtj6rEmAUXB20taKR3tYwyMkJiT1ekNCTuO824WU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/discrawl-kiwi/install-kiwi.sh b/tools/discrawl-kiwi/install-kiwi.sh new file mode 100644 index 00000000..bc3eb21e --- /dev/null +++ b/tools/discrawl-kiwi/install-kiwi.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +version=v0.23.2 +case "$(uname -s)" in + Darwin) platform=mac ;; + Linux) platform=lnx ;; + *) echo "unsupported Kiwi build platform: $(uname -s)" >&2; exit 1 ;; +esac +case "$(uname -m)" in + arm64|aarch64) + if [[ "$platform" == Linux ]]; then + architecture=aarch64 + else + architecture=arm64 + fi + ;; + x86_64|amd64) architecture=x86_64 ;; + *) echo "unsupported Kiwi architecture: $(uname -m)" >&2; exit 1 ;; +esac + +archive="kiwi_${platform}_${architecture}_${version}.tgz" +url="https://github.com/bab2min/Kiwi/releases/download/${version}/${archive}" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +curl --fail --location "$url" --output "$work/kiwi.tgz" +tar -xzf "$work/kiwi.tgz" -C "$work" +sudo cp -R "$work/include/kiwi" /usr/local/include/ +sudo cp -P "$work"/lib/libkiwi* /usr/local/lib/ +if [[ "$(uname -s)" == Linux ]]; then + sudo ldconfig +fi diff --git a/tools/discrawl-kiwi/main.go b/tools/discrawl-kiwi/main.go new file mode 100644 index 00000000..d99a49cc --- /dev/null +++ b/tools/discrawl-kiwi/main.go @@ -0,0 +1,95 @@ +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + + kiwi "github.com/codingpot/kiwigo" +) + +type request struct { + Text string `json:"text"` +} + +type response struct { + Ready bool `json:"ready,omitempty"` + Tokens string `json:"tokens,omitempty"` + Error string `json:"error,omitempty"` + Version string `json:"version,omitempty"` +} + +func main() { + model := flag.String("model", "", "path to the Kiwi base model directory") + flag.Parse() + if strings.TrimSpace(*model) == "" { + writeResponse(response{Error: "Kiwi model path is required; pass --model"}) + os.Exit(2) + } + analyzer, err := kiwi.New(*model, kiwi.WithNumThread(0)) + if err != nil { + writeResponse(response{Error: err.Error()}) + os.Exit(2) + } + defer analyzer.Close() + + writeResponse(response{Ready: true, Version: kiwi.KiwiVersion()}) + scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(make([]byte, 4096), 8*1024*1024) + for scanner.Scan() { + var input request + if err := json.Unmarshal(scanner.Bytes(), &input); err != nil { + writeResponse(response{Error: fmt.Sprintf("decode request: %v", err)}) + continue + } + tokens, err := tokenize(analyzer, input.Text) + if err != nil { + writeResponse(response{Error: err.Error()}) + continue + } + writeResponse(response{Tokens: strings.Join(tokens, " ")}) + } + if err := scanner.Err(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +type kiwiAnalyzer interface { + Analyze(string, ...kiwi.AnalyzeOptionFunc) ([]kiwi.TokenResult, error) +} + +func tokenize(analyzer kiwiAnalyzer, text string) ([]string, error) { + results, err := analyzer.Analyze(text, kiwi.WithTopN(1)) + if err != nil { + return nil, fmt.Errorf("analyze Korean text: %w", err) + } + if len(results) == 0 { + return nil, errors.New("Kiwi returned no analysis") + } + tokens := make([]string, 0, len(results[0].Tokens)) + seen := make(map[string]struct{}, len(results[0].Tokens)) + for _, token := range results[0].Tokens { + if strings.HasPrefix(string(token.Tag), "S") { + continue + } + form := strings.ToLower(strings.TrimSpace(token.Form)) + if form == "" { + continue + } + if _, ok := seen[form]; ok { + continue + } + seen[form] = struct{}{} + tokens = append(tokens, form) + } + return tokens, nil +} + +func writeResponse(output response) { + _ = json.NewEncoder(os.Stdout).Encode(output) +} diff --git a/tools/discrawl-kiwi/main_test.go b/tools/discrawl-kiwi/main_test.go new file mode 100644 index 00000000..aa64a212 --- /dev/null +++ b/tools/discrawl-kiwi/main_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "errors" + "strings" + "testing" + + kiwi "github.com/codingpot/kiwigo" + "github.com/stretchr/testify/require" +) + +type fakeAnalyzer struct { + results []kiwi.TokenResult + err error +} + +func (f fakeAnalyzer) Analyze(string, ...kiwi.AnalyzeOptionFunc) ([]kiwi.TokenResult, error) { + return f.results, f.err +} + +func TestTokenizeFiltersPunctuationAndDuplicates(t *testing.T) { + tokens, err := tokenize(fakeAnalyzer{results: []kiwi.TokenResult{{Tokens: []kiwi.TokenInfo{ + {Form: "오늘", Tag: kiwi.POS_NNG}, + {Form: ".", Tag: kiwi.POS_SF}, + {Form: "오늘", Tag: kiwi.POS_NNG}, + {Form: "먹", Tag: kiwi.POS_VV}, + }}}}, "오늘먹음") + require.NoError(t, err) + require.Equal(t, []string{"오늘", "먹"}, tokens) +} + +func TestTokenizeReportsAnalyzerFailures(t *testing.T) { + _, err := tokenize(fakeAnalyzer{err: errors.New("failed")}, "text") + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "failed")) + + _, err = tokenize(fakeAnalyzer{}, "text") + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "no analysis")) +} From 58e41fb96ec260b2d2a9094fa8bb8818df036cb8 Mon Sep 17 00:00:00 2001 From: "Jeffrey (Dongkyu) Kim" Date: Wed, 19 Aug 2026 16:42:47 +0900 Subject: [PATCH 4/5] refactor(search): drop Python lexical tokenizers Replace remaining ja/zh/ar Python workers with Go analyzers. Japanese and Chinese now run as optional Kagome and GSE helpers so the default binary stays small and CGO-free. Arabic light stemming is in-process. --- .github/workflows/ci.yml | 30 ++ THIRD_PARTY_NOTICES.md | 21 ++ docs/benchmarks/multilingual-lexical.md | 56 +-- docs/commands/lexical.md | 41 +-- docs/configuration.md | 11 +- docs/guides/search-modes.md | 26 +- internal/cli/cli.go | 53 +-- internal/cli/lexical_commands.go | 29 +- internal/cli/lexical_commands_test.go | 51 +-- internal/cli/output.go | 5 +- internal/config/config.go | 18 +- internal/config/config_test.go | 6 +- internal/store/lexical.go | 2 +- internal/store/lexical_arabic.go | 119 +++++++ internal/store/lexical_arabic_test.go | 32 ++ internal/store/lexical_helper.go | 253 ++++++++++++++ internal/store/lexical_helper_test.go | 152 ++++++++ internal/store/lexical_install.go | 136 -------- internal/store/lexical_kiwi.go | 191 +--------- internal/store/lexical_python.go | 327 ------------------ internal/store/lexical_python_command.go | 102 ------ internal/store/lexical_python_test.go | 210 ----------- internal/store/lexical_review_test.go | 267 ++------------ internal/store/lexical_status.go | 53 +++ internal/store/lexical_tokenizers.go | 74 ++++ internal/store/multilingual_benchmark_test.go | 3 +- internal/store/multilingual_e2e_test.go | 3 +- tools/discrawl-ja/README.md | 14 + tools/discrawl-ja/go.mod | 10 + tools/discrawl-ja/go.sum | 6 + tools/discrawl-ja/main.go | 81 +++++ tools/discrawl-ja/main_test.go | 26 ++ tools/discrawl-zh/README.md | 14 + tools/discrawl-zh/go.mod | 7 + tools/discrawl-zh/go.sum | 6 + tools/discrawl-zh/main.go | 66 ++++ tools/discrawl-zh/main_test.go | 25 ++ 37 files changed, 1143 insertions(+), 1383 deletions(-) create mode 100644 internal/store/lexical_arabic.go create mode 100644 internal/store/lexical_arabic_test.go create mode 100644 internal/store/lexical_helper.go create mode 100644 internal/store/lexical_helper_test.go delete mode 100644 internal/store/lexical_install.go delete mode 100644 internal/store/lexical_python.go delete mode 100644 internal/store/lexical_python_command.go delete mode 100644 internal/store/lexical_python_test.go create mode 100644 internal/store/lexical_status.go create mode 100644 internal/store/lexical_tokenizers.go create mode 100644 tools/discrawl-ja/README.md create mode 100644 tools/discrawl-ja/go.mod create mode 100644 tools/discrawl-ja/go.sum create mode 100644 tools/discrawl-ja/main.go create mode 100644 tools/discrawl-ja/main_test.go create mode 100644 tools/discrawl-zh/README.md create mode 100644 tools/discrawl-zh/go.mod create mode 100644 tools/discrawl-zh/go.sum create mode 100644 tools/discrawl-zh/main.go create mode 100644 tools/discrawl-zh/main_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb45d5b0..d5c57299 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -219,3 +219,33 @@ jobs: - name: Scan working tree run: | "$(go env GOPATH)/bin/gitleaks" dir . --no-banner --redact + + ja-helper: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: tools/discrawl-ja/go.mod + cache-dependency-path: tools/discrawl-ja/go.sum + - name: Test Japanese helper + working-directory: tools/discrawl-ja + run: go test -count=1 ./... + + zh-helper: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: tools/discrawl-zh/go.mod + cache-dependency-path: tools/discrawl-zh/go.sum + - name: Test Chinese helper + working-directory: tools/discrawl-zh + run: go test -count=1 ./... diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index fd89caaa..88962579 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -16,3 +16,24 @@ requirements. Kiwi's full license text is available from its source repository and the GNU project: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html + +## Optional Japanese lexical helper + +`tools/discrawl-ja` depends on: + +- [Kagome](https://github.com/ikawaha/kagome), MIT +- [kagome-dict IPA](https://github.com/ikawaha/kagome-dict), MIT wrapper around + mecab-ipadic-2.7.0-20070801 / ICOT Free Software + +These dependencies are optional and are not linked into the default Discrawl +binary. Preserve the IPADIC/ICOT notice when distributing the helper. + +## Optional Chinese lexical helper + +`tools/discrawl-zh` depends on [GSE](https://github.com/go-ego/gse), Apache-2.0. +It is optional and is not linked into the default Discrawl binary. + +## In-process Arabic analyzer + +The Arabic light stemmer follows the Lucene/Bleve prefix-and-suffix contract. +It is implemented in Discrawl itself and does not depend on Python. diff --git a/docs/benchmarks/multilingual-lexical.md b/docs/benchmarks/multilingual-lexical.md index 7cfdc87d..db6cf949 100644 --- a/docs/benchmarks/multilingual-lexical.md +++ b/docs/benchmarks/multilingual-lexical.md @@ -11,17 +11,16 @@ cannot retrieve as independent terms. ## Reproduce ```bash -python3 -m venv /tmp/discrawl-tokenizer-e2e -/tmp/discrawl-tokenizer-e2e/bin/python -m pip install \ - sudachipy==0.6.11 \ - sudachidict_core==20260723 \ - jieba==0.42.1 \ - snowballstemmer==3.1.1 +# Korean helper: official Kiwi 0.23.2 + discrawl-kiwi +# Japanese helper: go build ./tools/discrawl-ja +# Chinese helper: go build ./tools/discrawl-zh +# Arabic: in-process, no helper DISCRAWL_TOKENIZER_E2E=1 \ -DISCRAWL_TOKENIZER_PYTHON=/tmp/discrawl-tokenizer-e2e/bin/python \ DISCRAWL_KIWI_HELPER=/tmp/discrawl-kiwi \ DISCRAWL_KIWI_MODEL=/tmp/kiwi-model/models/cong/base \ +DISCRAWL_JA_HELPER=/tmp/discrawl-ja \ +DISCRAWL_ZH_HELPER=/tmp/discrawl-zh \ go test ./internal/store \ -run TestMultilingualLexicalQualityBenchmark \ -count=1 -v @@ -29,45 +28,18 @@ go test ./internal/store \ ## Result -Measured on macOS arm64 with Kiwi 0.23.2 through `kiwigo`; the remaining -analyzers used Python 3.13.2: +Measured on macOS arm64 with native Kiwi 0.23.2, Kagome Search, GSE CutSearch, +and the in-process Arabic analyzer: | Language | `unicode61` recall@5 | Multilingual recall@5 | | --- | ---: | ---: | -| Korean / Kiwi | 0/5 | 5/5 | -| Japanese / Sudachi A | 0/5 | 5/5 | -| Chinese / Jieba search mode | 0/5 | 5/5 | -| Arabic / Snowball + proclitics | 0/5 | 5/5 | +| Korean / Kiwi via kiwigo | 0/5 | 5/5 | +| Japanese / Kagome Search | 0/5 | 5/5 | +| Chinese / GSE search mode | 0/5 | 5/5 | +| Arabic / in-process light stem | 0/5 | 5/5 | | **Total** | **0/20** | **20/20** | -Across repeated runs of this 20-message fixture, the database with four extra -FTS tables used 1.36-1.37x the SQLite pages of the baseline. Real corpora will -have different ratios because base tables, attachments, and metadata are not -duplicated while postings scale with enabled analyzers. - The benchmark therefore supports a narrow claim: configured language fields substantially improve recall for these segmentation cases. It does not claim a -universal 100-point gain on natural Discord query distributions. - -## Live verification transcript - -Captured from the command above: - -```text -=== RUN TestMultilingualLexicalQualityBenchmark -database pages: unicode61=233472 bytes multilingual=319488 bytes (1.37x) -ko recall@5: unicode61=0/5 multilingual=5/5 -ja recall@5: unicode61=0/5 multilingual=5/5 -zh recall@5: unicode61=0/5 multilingual=5/5 -ar recall@5: unicode61=0/5 multilingual=5/5 ---- PASS: TestMultilingualLexicalQualityBenchmark -=== RUN TestMultilingualLexicalSearchE2E ---- PASS: TestMultilingualLexicalSearchE2E -PASS -``` - -The Korean path now uses the published `github.com/codingpot/kiwigo` Go binding -against Kiwi 0.23.2. The live helper returned `오늘 저녁 먹 음 기록` for -`오늘 저녁먹음 기록`; a subsequent CLI search for `저녁` returned that fixture. -No Python process or `kiwipiepy` package is involved in Korean indexing or -query analysis. +universal 100-point gain on natural Discord query distributions. No Python +tokenizer is used. diff --git a/docs/commands/lexical.md b/docs/commands/lexical.md index b28806dd..0198cd87 100644 --- a/docs/commands/lexical.md +++ b/docs/commands/lexical.md @@ -1,8 +1,7 @@ # `lexical` -Installs only the optional Python tokenizer packages selected by -`search.lexical.languages`. Korean is provided by the native -`discrawl-kiwi` Go helper and is not installed by this command. +Shows the Go helpers required by `search.lexical.languages`. Discrawl never +downloads or installs tokenizer packages. ## Usage @@ -10,34 +9,24 @@ Installs only the optional Python tokenizer packages selected by discrawl lexical install ``` -The configured `search.lexical.python` interpreter must belong to a virtual -environment when Japanese, Chinese, or Arabic is selected. Discrawl refuses to -install packages into a system Python. With a Korean-only configuration this -command is a no-op because Korean uses the separately built Go helper. +| Language | Runtime | Command | +| --- | --- | --- | +| `ko` | helper | `discrawl-kiwi` (`github.com/codingpot/kiwigo` + Kiwi 0.23.2) | +| `ja` | helper | `discrawl-ja` (`github.com/ikawaha/kagome/v2` Search) | +| `zh` | helper | `discrawl-zh` (`github.com/go-ego/gse` CutSearch) | +| `ar` | in-process | none | -Package versions are pinned by Discrawl: - -| Language | Packages | -| --- | --- | -| `ko` | none; uses `github.com/codingpot/kiwigo` + Kiwi 0.23.2 | -| `ja` | `sudachipy==0.6.11`, `sudachidict_core==20260723` | -| `zh` | `jieba==0.42.1` | -| `ar` | `snowballstemmer==3.1.1` | - -Installation is always explicit. Opening an archive or running a search never -downloads or installs code. Tokenizer workers start lazily only when an enabled -language is first used for indexing or search. - -For Korean, configure: +Tokenizer helpers start lazily only when an enabled language is first used for +indexing or search. Opening an archive never starts a helper. ```toml [search.lexical] -languages = ["ko"] +languages = ["ko", "ja", "zh", "ar"] kiwi_command = "~/.local/share/discrawl/bin/discrawl-kiwi" kiwi_model = "~/.local/share/discrawl/models/kiwi/base" +ja_command = "~/.local/share/discrawl/bin/discrawl-ja" +zh_command = "~/.local/share/discrawl/bin/discrawl-zh" ``` -The helper source is under `tools/discrawl-kiwi`. It uses the existing -[`github.com/codingpot/kiwigo`](https://pkg.go.dev/github.com/codingpot/kiwigo) -binding and dynamically linked Kiwi library; it does not use Python or -`kiwipiepy`. +Helper sources live under `tools/discrawl-kiwi`, `tools/discrawl-ja`, and +`tools/discrawl-zh`. None of those helpers use Python. diff --git a/docs/configuration.md b/docs/configuration.md index 80beb358..52884fdc 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -79,9 +79,10 @@ default_mode = "fts" [search.lexical] languages = [] # optional: "ko", "ja", "zh", "ar" -python = "python3" kiwi_command = "discrawl-kiwi" kiwi_model = "" +ja_command = "discrawl-ja" +zh_command = "discrawl-zh" [search.embeddings] enabled = false @@ -146,10 +147,10 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and - `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion - `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement` - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time -- `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi through the `github.com/codingpot/kiwigo` Go binding), Japanese (`ja`, Sudachi), Chinese (`zh`, Jieba), and Arabic (`ar`, Snowball plus proclitic splitting). -- Korean requires the separately built `discrawl-kiwi` helper plus Kiwi 0.23.2's dynamic library and base model. Set `kiwi_command` and `kiwi_model` to their installed paths. Korean does not use Python or `kiwipiepy`. -- `discrawl lexical install` installs only the pinned Python packages required by configured Japanese, Chinese, or Arabic fields and refuses system-Python installation. Search and sync never install packages implicitly. -- Tokenizer workers load lazily on first indexing or search use. Commands that only inspect metadata do not start the Kiwi helper or Python modules. +- `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi through `github.com/codingpot/kiwigo`), Japanese (`ja`, Kagome Search through `discrawl-ja`), Chinese (`zh`, GSE CutSearch through `discrawl-zh`), and Arabic (`ar`, in-process light stemming). +- Korean, Japanese, and Chinese use separately built Go helpers so the default Discrawl binary stays small and CGO-free. Arabic is implemented in-process. +- `discrawl lexical install` reports the required helpers. It never downloads packages. +- Tokenizer helpers load lazily on first indexing or search use. Commands that only inspect metadata do not start helpers. - Each enabled language adds an independent FTS5 table. Index and query text pass through the same tokenizer, and results from the default plus language-specific tables are merged with reciprocal rank fusion. - After adding or changing `search.lexical.languages`, run a writer command such as `discrawl sync` once so the configured lexical tables are built. Read-only commands never mutate the archive; new and edited messages update the tables automatically during later syncs. - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search diff --git a/docs/guides/search-modes.md b/docs/guides/search-modes.md index eb844edc..93bbc155 100644 --- a/docs/guides/search-modes.md +++ b/docs/guides/search-modes.md @@ -12,7 +12,7 @@ - backed by SQLite FTS5 with the default `unicode61` tokenizer - optional `[search.lexical]` languages add independent tokenizer-specific FTS tables and merge their ranked results with reciprocal rank fusion -- supported presets are Korean with the native Kiwi engine through the `kiwigo` Go binding, Japanese with Sudachi A-mode, Chinese with Jieba search mode, and Arabic with Snowball stemming plus proclitic splitting +- supported presets are Korean with native Kiwi through `kiwigo`, Japanese with Kagome Search, Chinese with GSE search mode, and Arabic with in-process light stemming - user query terms are parameterized and quoted before `MATCH`, so tokens like `AND`, `OR`, `NOT`, `NEAR`, and `*` are searched as input terms instead of FTS operators - punctuation still follows FTS5 tokenization rules - by default, `search` skips rows with no searchable content (attachment text, attachment filenames, embeds, and replies still count as content); use `--include-empty` to opt back in @@ -31,11 +31,11 @@ go build -o ~/.local/share/discrawl/bin/discrawl-kiwi . installation expects Kiwi headers and dynamic libraries under `/usr/local`; the model is the `kiwi_model_v0.23.2_base.tgz` release asset. -Create an isolated Python environment only if Japanese, Chinese, or Arabic is -enabled: +Build the optional Japanese and Chinese helpers: ```bash -python3 -m venv ~/.local/share/discrawl/tokenizers +cd tools/discrawl-ja && go build -o ~/.local/share/discrawl/bin/discrawl-ja . +cd ../discrawl-zh && go build -o ~/.local/share/discrawl/bin/discrawl-zh . ``` Configure the fields: @@ -43,16 +43,14 @@ Configure the fields: ```toml [search.lexical] languages = ["ko", "ja", "zh", "ar"] -python = "~/.local/share/discrawl/tokenizers/bin/python" # ~ is expanded kiwi_command = "~/.local/share/discrawl/bin/discrawl-kiwi" kiwi_model = "~/.local/share/discrawl/models/kiwi/base" +ja_command = "~/.local/share/discrawl/bin/discrawl-ja" +zh_command = "~/.local/share/discrawl/bin/discrawl-zh" ``` -Install only the Python packages selected by non-Korean languages: - -```bash -discrawl lexical install -``` +`discrawl lexical install` reports those helper paths. It does not download +packages. Every message is analyzed into each configured field. This deliberately avoids language detection, so mixed-language Discord messages remain searchable @@ -60,11 +58,9 @@ through every enabled analyzer. Disk usage and indexing work increase with the number of fields; query-time RRF deduplicates message ids without mixing the different BM25 term statistics into one field. -Korean text never crosses a Python boundary: `discrawl-kiwi` is a persistent -Go helper built against `github.com/codingpot/kiwigo` and dynamically linked to -Kiwi. Discrawl never installs packages during archive open, sync, or search. -All helpers are loaded lazily on first use, while `lexical install` is an -explicit, virtual-environment-only operation for the remaining analyzers. +None of the lexical analyzers use Python. Korean, Japanese, and Chinese run as +persistent Go helpers; Arabic is in-process. Helpers load lazily on first use +and are not linked into the default Discrawl binary. See [Multilingual lexical benchmark](../benchmarks/multilingual-lexical.html) for the reproducible targeted quality check and its storage tradeoff. diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 24f83173..f2d0e87f 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -128,7 +128,7 @@ var discrawlCommandSpecs = []discrawlCommandSpec{ {name: "cache-import", description: "Import Discord Desktop cache data (wiretap alias)."}, {name: "wiretap", description: "Import Discord Desktop cache data."}, {name: "search", description: "Search archived messages."}, - {name: "lexical", description: "Install configured Python lexical tokenizers."}, + {name: "lexical", description: "Show configured Go lexical helpers."}, {name: "tui", description: "Explore the archive in an interactive terminal UI."}, {name: "messages", description: "List archived messages."}, {name: "digest", description: "Summarize recent archive activity."}, @@ -249,29 +249,28 @@ func parseKongArgs(target any, args []string, name string, stdout, stderr io.Wri } type runtime struct { - ctx context.Context - configPath string - cfg config.Config - stdout io.Writer - stderr io.Writer - json bool - plain bool - logger *slog.Logger - store *store.Store - client discordClient - syncer syncService - dbLockHeld bool - lockStarted time.Time - lockOperation string - lockToken string - lockTokenFree func() error - openStore func(context.Context, string) (*store.Store, error) - newDiscord func(config.Config) (discordClient, error) - newRemote func(config.Config) (remoteArchiveClient, error) - newSyncer func(syncer.Client, *store.Store, *slog.Logger) syncService - newEmbed func(config.EmbeddingsConfig) (embed.Provider, error) - installLexical func(context.Context, string, []string) (store.LexicalInstallResult, error) - now func() time.Time + ctx context.Context + configPath string + cfg config.Config + stdout io.Writer + stderr io.Writer + json bool + plain bool + logger *slog.Logger + store *store.Store + client discordClient + syncer syncService + dbLockHeld bool + lockStarted time.Time + lockOperation string + lockToken string + lockTokenFree func() error + openStore func(context.Context, string) (*store.Store, error) + newDiscord func(config.Config) (discordClient, error) + newRemote func(config.Config) (remoteArchiveClient, error) + newSyncer func(syncer.Client, *store.Store, *slog.Logger) syncService + newEmbed func(config.EmbeddingsConfig) (embed.Provider, error) + now func() time.Time } func crawlkitEmbeddingConfig(cfg config.EmbeddingsConfig) embed.Config { @@ -695,9 +694,10 @@ func (r *runtime) openConfiguredReadOnlyStore(path string) (*store.Store, error) } return store.OpenReadOnlyWithOptions(r.ctx, path, store.OpenOptions{ LexicalLanguages: r.cfg.Search.Lexical.Languages, - LexicalPython: r.cfg.Search.Lexical.Python, LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand, LexicalKiwiModel: r.cfg.Search.Lexical.KiwiModel, + LexicalJaCommand: r.cfg.Search.Lexical.JaCommand, + LexicalZhCommand: r.cfg.Search.Lexical.ZhCommand, }) } @@ -774,9 +774,10 @@ func (r *runtime) localStoreFactory() func(context.Context, string) (*store.Stor return func(ctx context.Context, path string) (*store.Store, error) { return store.OpenWithOptions(ctx, path, store.OpenOptions{ LexicalLanguages: r.cfg.Search.Lexical.Languages, - LexicalPython: r.cfg.Search.Lexical.Python, LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand, LexicalKiwiModel: r.cfg.Search.Lexical.KiwiModel, + LexicalJaCommand: r.cfg.Search.Lexical.JaCommand, + LexicalZhCommand: r.cfg.Search.Lexical.ZhCommand, }) } } diff --git a/internal/cli/lexical_commands.go b/internal/cli/lexical_commands.go index e4b3954c..9aa603a5 100644 --- a/internal/cli/lexical_commands.go +++ b/internal/cli/lexical_commands.go @@ -2,16 +2,13 @@ package cli import ( "errors" - "fmt" "github.com/openclaw/discrawl/internal/store" ) type lexicalInstallOutput struct { - Languages []string `json:"languages"` - Packages []string `json:"packages"` - Python string `json:"python"` - Kiwi string `json:"kiwi,omitempty"` + Languages []string `json:"languages"` + Helpers []store.LexicalHelperStatus `json:"helpers"` } func (r *runtime) runLexical(args []string) error { @@ -21,22 +18,14 @@ func (r *runtime) runLexical(args []string) error { if len(r.cfg.Search.Lexical.Languages) == 0 { return configErr(errors.New("search.lexical.languages is empty")) } - install := r.installLexical - if install == nil { - install = store.InstallLexicalPackages - } - result, err := install( - r.ctx, - r.cfg.Search.Lexical.Python, - r.cfg.Search.Lexical.Languages, - ) - if err != nil { - return configErr(fmt.Errorf("install lexical tokenizers: %w", err)) - } return r.print(lexicalInstallOutput{ Languages: append([]string(nil), r.cfg.Search.Lexical.Languages...), - Packages: append([]string{}, result.Packages...), - Python: r.cfg.Search.Lexical.Python, - Kiwi: r.cfg.Search.Lexical.KiwiCommand, + Helpers: store.LexicalHelperStatuses(store.OpenOptions{ + LexicalLanguages: r.cfg.Search.Lexical.Languages, + LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand, + LexicalKiwiModel: r.cfg.Search.Lexical.KiwiModel, + LexicalJaCommand: r.cfg.Search.Lexical.JaCommand, + LexicalZhCommand: r.cfg.Search.Lexical.ZhCommand, + }), }) } diff --git a/internal/cli/lexical_commands_test.go b/internal/cli/lexical_commands_test.go index 09c76c7a..d66cfe46 100644 --- a/internal/cli/lexical_commands_test.go +++ b/internal/cli/lexical_commands_test.go @@ -3,17 +3,15 @@ package cli import ( "bytes" "context" - "errors" "os" "path/filepath" "testing" "github.com/openclaw/discrawl/internal/config" - "github.com/openclaw/discrawl/internal/store" "github.com/stretchr/testify/require" ) -func TestRunLexicalInstallUsesConfiguredLanguages(t *testing.T) { +func TestRunLexicalInstallReportsConfiguredHelpers(t *testing.T) { var stdout bytes.Buffer r := &runtime{ ctx: context.Background(), @@ -21,49 +19,30 @@ func TestRunLexicalInstallUsesConfiguredLanguages(t *testing.T) { stdout: &stdout, } r.cfg.Search.Lexical.Languages = []string{"ko", "zh"} - r.cfg.Search.Lexical.Python = "/tmp/tokenizers/bin/python" - r.installLexical = func( - _ context.Context, - python string, - languages []string, - ) (store.LexicalInstallResult, error) { - require.Equal(t, "/tmp/tokenizers/bin/python", python) - require.Equal(t, []string{"ko", "zh"}, languages) - return store.LexicalInstallResult{ - Packages: []string{"jieba==0.42.1"}, - }, nil - } + r.cfg.Search.Lexical.ZhCommand = "/tmp/discrawl-zh" require.NoError(t, r.runLexical([]string{"install"})) - require.Contains(t, stdout.String(), "jieba==0.42.1") + require.Contains(t, stdout.String(), "discrawl-kiwi") + require.Contains(t, stdout.String(), "/tmp/discrawl-zh") + require.NotContains(t, stdout.String(), "python") } func TestRunLexicalInstallRequiresConfiguredLanguages(t *testing.T) { r := &runtime{ ctx: context.Background(), cfg: config.Default(), - installLexical: func(context.Context, string, []string) (store.LexicalInstallResult, error) { - return store.LexicalInstallResult{}, errors.New("must not run") - }, } - err := r.runLexical([]string{"install"}) require.ErrorContains(t, err, "search.lexical.languages is empty") } -func TestRunLexicalInstallReportsInstallerError(t *testing.T) { +func TestRunLexicalInstallReportsUsage(t *testing.T) { r := &runtime{ ctx: context.Background(), cfg: config.Default(), - installLexical: func(context.Context, string, []string) (store.LexicalInstallResult, error) { - return store.LexicalInstallResult{}, errors.New("pip unavailable") - }, } r.cfg.Search.Lexical.Languages = []string{"ko"} - - err := r.runLexical([]string{"install"}) - require.ErrorContains(t, err, "pip unavailable") - err = r.runLexical([]string{"unknown"}) + err := r.runLexical([]string{"unknown"}) require.ErrorContains(t, err, "usage: discrawl lexical install") } @@ -74,18 +53,16 @@ func TestRunLexicalInstallJSONOutput(t *testing.T) { cfg: config.Default(), stdout: &stdout, json: true, - installLexical: func(context.Context, string, []string) (store.LexicalInstallResult, error) { - return store.LexicalInstallResult{}, nil - }, } - r.cfg.Search.Lexical.Languages = []string{"ko"} + r.cfg.Search.Lexical.Languages = []string{"ko", "ar"} require.NoError(t, r.runLexical([]string{"install"})) require.JSONEq(t, `{ - "languages": ["ko"], - "packages": [], - "python": "python3", - "kiwi": "discrawl-kiwi" + "languages": ["ko", "ar"], + "helpers": [ + {"language":"ko","runtime":"helper","command":"discrawl-kiwi"}, + {"language":"ar","runtime":"in-process"} + ] }`, stdout.String()) } @@ -93,6 +70,8 @@ func TestLexicalHelp(t *testing.T) { var stdout bytes.Buffer require.NoError(t, Run(context.Background(), []string{"help", "lexical"}, &stdout, &bytes.Buffer{})) require.Contains(t, stdout.String(), "discrawl lexical install") + require.Contains(t, stdout.String(), "discrawl-ja") + require.NotContains(t, stdout.String(), "Python packages") } func TestRunDispatchesLexicalInstall(t *testing.T) { diff --git a/internal/cli/output.go b/internal/cli/output.go index 3d2cc6d5..04f78978 100644 --- a/internal/cli/output.go +++ b/internal/cli/output.go @@ -252,8 +252,9 @@ Flags: "lexical": `Usage: discrawl lexical install -Install pinned Python packages for configured Japanese, Chinese, and Arabic fields. -Korean uses the separately built discrawl-kiwi Go helper and requires no Python package. +Show the Go helpers required by search.lexical.languages. +Korean uses discrawl-kiwi, Japanese uses discrawl-ja, Chinese uses discrawl-zh, and Arabic is in-process. +Discrawl never downloads tokenizers. `, "attachments": `Usage: discrawl attachments [flags] diff --git a/internal/config/config.go b/internal/config/config.go index a490b4bc..a7135bf9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -74,9 +74,10 @@ type SearchConfig struct { type LexicalSearchConfig struct { Languages []string `toml:"languages,omitempty"` - Python string `toml:"python"` KiwiCommand string `toml:"kiwi_command"` KiwiModel string `toml:"kiwi_model"` + JaCommand string `toml:"ja_command"` + ZhCommand string `toml:"zh_command"` } type ShareConfig struct { @@ -163,8 +164,9 @@ func Default() Config { Search: SearchConfig{ DefaultMode: "fts", Lexical: LexicalSearchConfig{ - Python: "python3", KiwiCommand: "discrawl-kiwi", + JaCommand: "discrawl-ja", + ZhCommand: "discrawl-zh", }, Embeddings: EmbeddingsConfig{ Enabled: false, @@ -312,15 +314,19 @@ func (c *Config) Normalize() error { if c.Search.DefaultMode == "" { c.Search.DefaultMode = "fts" } - c.Search.Lexical.Python = strings.TrimSpace(c.Search.Lexical.Python) - if c.Search.Lexical.Python == "" { - c.Search.Lexical.Python = "python3" - } c.Search.Lexical.KiwiCommand = strings.TrimSpace(c.Search.Lexical.KiwiCommand) if c.Search.Lexical.KiwiCommand == "" { c.Search.Lexical.KiwiCommand = "discrawl-kiwi" } c.Search.Lexical.KiwiModel = strings.TrimSpace(c.Search.Lexical.KiwiModel) + c.Search.Lexical.JaCommand = strings.TrimSpace(c.Search.Lexical.JaCommand) + if c.Search.Lexical.JaCommand == "" { + c.Search.Lexical.JaCommand = "discrawl-ja" + } + c.Search.Lexical.ZhCommand = strings.TrimSpace(c.Search.Lexical.ZhCommand) + if c.Search.Lexical.ZhCommand == "" { + c.Search.Lexical.ZhCommand = "discrawl-zh" + } seenLexicalLanguages := make(map[string]struct{}, len(c.Search.Lexical.Languages)) normalizedLexicalLanguages := make([]string, 0, len(c.Search.Lexical.Languages)) for _, language := range c.Search.Lexical.Languages { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5544b2fa..a3c5f327 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -261,17 +261,19 @@ token_source = "env" [search.lexical] languages = ["ko", "ja", "zh", "ar"] -python = "/opt/discrawl-tokenizers/bin/python" kiwi_command = "/opt/discrawl/bin/discrawl-kiwi" kiwi_model = "/opt/discrawl/models/kiwi/base" +ja_command = "/opt/discrawl/bin/discrawl-ja" +zh_command = "/opt/discrawl/bin/discrawl-zh" `), 0o600)) cfg, err := Load(path) require.NoError(t, err) require.Equal(t, []string{"ko", "ja", "zh", "ar"}, cfg.Search.Lexical.Languages) - require.Equal(t, "/opt/discrawl-tokenizers/bin/python", cfg.Search.Lexical.Python) require.Equal(t, "/opt/discrawl/bin/discrawl-kiwi", cfg.Search.Lexical.KiwiCommand) require.Equal(t, "/opt/discrawl/models/kiwi/base", cfg.Search.Lexical.KiwiModel) + require.Equal(t, "/opt/discrawl/bin/discrawl-ja", cfg.Search.Lexical.JaCommand) + require.Equal(t, "/opt/discrawl/bin/discrawl-zh", cfg.Search.Lexical.ZhCommand) } func TestNormalizeRejectsUnsupportedLexicalLanguage(t *testing.T) { diff --git a/internal/store/lexical.go b/internal/store/lexical.go index 86b8417d..f6103829 100644 --- a/internal/store/lexical.go +++ b/internal/store/lexical.go @@ -9,7 +9,7 @@ import ( "time" ) -const lexicalFTSVersion = "1" +const lexicalFTSVersion = "2" type LexicalTokenizer interface { Tokenize(context.Context, string) (string, error) diff --git a/internal/store/lexical_arabic.go b/internal/store/lexical_arabic.go new file mode 100644 index 00000000..40be3fff --- /dev/null +++ b/internal/store/lexical_arabic.go @@ -0,0 +1,119 @@ +package store + +import ( + "context" + "strings" + "unicode" + "unicode/utf8" +) + +// Arabic light stemming follows the Lucene/Bleve prefix-and-suffix +// contract so attached proclitics remain searchable as independent terms. +var arabicPrefixes = []string{"وال", "فال", "بال", "كال", "لال", "ال", "لل", "و", "ف", "ب", "ك", "ل"} + +var arabicSuffixes = []string{"ها", "ان", "ات", "ون", "ين", "يه", "ية", "ه", "ة", "ي"} + +type arabicLexicalTokenizer struct{} + +func newArabicLexicalTokenizer() LexicalTokenizer { + return arabicLexicalTokenizer{} +} + +func (arabicLexicalTokenizer) Tokenize(_ context.Context, text string) (string, error) { + return strings.Join(tokenizeArabic(text), " "), nil +} + +func (arabicLexicalTokenizer) Close() error { + return nil +} + +func tokenizeArabic(text string) []string { + seen := make(map[string]struct{}) + var tokens []string + add := func(token string) { + token = strings.ToLower(strings.TrimSpace(token)) + if token == "" { + return + } + if _, ok := seen[token]; ok { + return + } + seen[token] = struct{}{} + tokens = append(tokens, token) + } + for _, word := range splitArabicWords(normalizeArabic(text)) { + add(word) + stripped := stripArabicPrefix(word) + add(stripped) + add(stemArabic(word)) + add(stemArabic(stripped)) + } + return tokens +} + +func splitArabicWords(text string) []string { + var words []string + var current []rune + flush := func() { + if len(current) == 0 { + return + } + words = append(words, string(current)) + current = current[:0] + } + for _, r := range text { + if unicode.IsLetter(r) { + current = append(current, r) + continue + } + flush() + } + flush() + return words +} + +func normalizeArabic(text string) string { + var b strings.Builder + b.Grow(len(text)) + for _, r := range strings.ToValidUTF8(text, "") { + switch r { + case '\u064B', '\u064C', '\u064D', '\u064E', '\u064F', '\u0650', '\u0651', '\u0652', '\u0670', '\u0640': + continue + default: + b.WriteRune(unicode.ToLower(r)) + } + } + return b.String() +} + +func stripArabicPrefix(word string) string { + for _, prefix := range arabicPrefixes { + if !strings.HasPrefix(word, prefix) { + continue + } + rest := strings.TrimPrefix(word, prefix) + if utf8.RuneCountInString(rest) < 2 { + continue + } + if prefix == "و" && utf8.RuneCountInString(word) < 4 { + continue + } + return rest + } + return word +} + +func stemArabic(word string) string { + runes := []rune(stripArabicPrefix(word)) + for _, suffix := range arabicSuffixes { + suffixRunes := []rune(suffix) + if len(runes)-len(suffixRunes) < 2 { + continue + } + if string(runes[len(runes)-len(suffixRunes):]) != suffix { + continue + } + runes = runes[:len(runes)-len(suffixRunes)] + } + return string(runes) +} diff --git a/internal/store/lexical_arabic_test.go b/internal/store/lexical_arabic_test.go new file mode 100644 index 00000000..8ab19482 --- /dev/null +++ b/internal/store/lexical_arabic_test.go @@ -0,0 +1,32 @@ +package store + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTokenizeArabicSplitsProcliticsForSearch(t *testing.T) { + cases := map[string]string{ + "والكتاب": "كتاب", + "والمدرسة": "مدرسة", + "فالاجتماع": "اجتماع", + "بالسجل": "سجل", + "كالبرنامج": "برنامج", + } + for input, want := range cases { + tokens := tokenizeArabic(input) + require.Containsf(t, tokens, want, "input %q tokens=%v", input, tokens) + } +} + +func TestArabicTokenizerIsInProcessAndIdempotent(t *testing.T) { + tokenizer := newArabicLexicalTokenizer() + tokens, err := tokenizer.Tokenize(context.Background(), "والكتاب في المدرسة") + require.NoError(t, err) + require.Contains(t, strings.Split(tokens, " "), "كتاب") + require.NoError(t, tokenizer.Close()) + require.NoError(t, tokenizer.Close()) +} diff --git a/internal/store/lexical_helper.go b/internal/store/lexical_helper.go new file mode 100644 index 00000000..98a5a0b3 --- /dev/null +++ b/internal/store/lexical_helper.go @@ -0,0 +1,253 @@ +package store + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type helperLexicalResponse struct { + Ready bool `json:"ready,omitempty"` + Tokens string `json:"tokens,omitempty"` + Error string `json:"error,omitempty"` +} + +type lockedBuffer struct { + mutex sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(data []byte) (int, error) { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.buffer.Write(data) +} + +func (b *lockedBuffer) String() string { + b.mutex.Lock() + defer b.mutex.Unlock() + return b.buffer.String() +} + +func startHelperLexicalTokenizer(language, command, defaultName string, extraArgs []string) (LexicalTokenizer, error) { + cmd, err := newHelperLexicalCommand(command, defaultName, extraArgs) + if err != nil { + return nil, err + } + return startHelperLexicalTokenizerCommand(language, cmd) +} + +func startHelperLexicalTokenizerCommand(language string, cmd *exec.Cmd) (LexicalTokenizer, error) { + tokenizer := &externalLexicalTokenizer{ + language: language, + command: cmd, + } + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + _ = stdin.Close() + return nil, err + } + cmd.Stderr = &tokenizer.stderr + tokenizer.stdin = stdin + tokenizer.stdout = bufio.NewScanner(stdout) + tokenizer.stdout.Buffer(make([]byte, 4096), 8*1024*1024) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("start %s lexical tokenizer: %w", language, err) + } + response, err := tokenizer.readStartup() + if err != nil { + _ = tokenizer.Close() + return nil, fmt.Errorf("initialize %s lexical tokenizer: %w", language, err) + } + if !response.Ready { + _ = tokenizer.Close() + if response.Error == "" { + return nil, fmt.Errorf("initialize %s lexical tokenizer", language) + } + return nil, errors.New(response.Error) + } + return tokenizer, nil +} + +func newHelperLexicalCommand(command, defaultName string, extraArgs []string) (*exec.Cmd, error) { + command, err := expandLexicalPath(command, defaultName) + if err != nil { + return nil, fmt.Errorf("resolve %s helper: %w", defaultName, err) + } + if !filepath.IsAbs(command) { + if command != defaultName { + return nil, fmt.Errorf( + "unsupported %s helper %q; use an absolute path or %s", + defaultName, + command, + defaultName, + ) + } + command, err = exec.LookPath(command) + if err != nil { + return nil, fmt.Errorf("find %s helper: %w", defaultName, err) + } + } + args := append([]string{command}, extraArgs...) + return &exec.Cmd{ + Path: command, + Args: args, + Env: lexicalWorkerEnvironment(os.Environ()), + }, nil +} + +func expandLexicalPath(path, fallback string) (string, error) { + path = strings.TrimSpace(path) + if path == "" { + return fallback, nil + } + if !strings.HasPrefix(path, "~/") { + return path, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, strings.TrimPrefix(path, "~/")), nil +} + +type externalLexicalTokenizer struct { + language string + command *exec.Cmd + stdin io.WriteCloser + stdout *bufio.Scanner + stderr lockedBuffer + mutex sync.Mutex +} + +func (t *externalLexicalTokenizer) readStartup() (helperLexicalResponse, error) { + result := make(chan struct { + response helperLexicalResponse + err error + }, 1) + go func() { + response, err := t.readResponse() + result <- struct { + response helperLexicalResponse + err error + }{response, err} + }() + select { + case output := <-result: + return output.response, output.err + case <-time.After(30 * time.Second): + _ = t.command.Process.Kill() + return helperLexicalResponse{}, errors.New("lexical tokenizer startup timed out after 30s") + } +} + +func (t *externalLexicalTokenizer) Tokenize(ctx context.Context, text string) (string, error) { + t.mutex.Lock() + defer t.mutex.Unlock() + if err := ctx.Err(); err != nil { + return "", err + } + request, err := json.Marshal(map[string]string{"text": text}) + if err != nil { + return "", err + } + if _, err := t.stdin.Write(append(request, '\n')); err != nil { + return "", fmt.Errorf("write %s tokenizer request: %w", t.language, err) + } + result := make(chan struct { + response helperLexicalResponse + err error + }, 1) + go func() { + response, err := t.readResponse() + result <- struct { + response helperLexicalResponse + err error + }{response, err} + }() + select { + case output := <-result: + if output.err != nil { + return "", output.err + } + if output.response.Error != "" { + return "", errors.New(output.response.Error) + } + return output.response.Tokens, nil + case <-ctx.Done(): + _ = t.command.Process.Kill() + return "", ctx.Err() + case <-time.After(30 * time.Second): + _ = t.command.Process.Kill() + return "", errors.New("lexical tokenizer response timed out after 30s") + } +} + +func (t *externalLexicalTokenizer) readResponse() (helperLexicalResponse, error) { + if !t.stdout.Scan() { + if err := t.stdout.Err(); err != nil { + return helperLexicalResponse{}, err + } + if detail := strings.TrimSpace(t.stderr.String()); detail != "" { + return helperLexicalResponse{}, errors.New(detail) + } + return helperLexicalResponse{}, io.EOF + } + var response helperLexicalResponse + if err := json.Unmarshal(t.stdout.Bytes(), &response); err != nil { + return helperLexicalResponse{}, fmt.Errorf("decode lexical tokenizer response: %w", err) + } + return response, nil +} + +func (t *externalLexicalTokenizer) Close() error { + if t == nil || t.command == nil || t.command.Process == nil { + return nil + } + _ = t.stdin.Close() + err := t.command.Wait() + if errors.Is(err, os.ErrProcessDone) { + return nil + } + return err +} + +func lexicalWorkerEnvironment(parent []string) []string { + allowed := map[string]struct{}{ + "HOME": {}, + "LANG": {}, + "LC_ALL": {}, + "PATH": {}, + "PATHEXT": {}, + "SYSTEMROOT": {}, + "TEMP": {}, + "TMP": {}, + "TMPDIR": {}, + "WINDIR": {}, + } + environment := make([]string, 0, len(allowed)) + for _, entry := range parent { + key, _, ok := strings.Cut(entry, "=") + if !ok { + continue + } + if _, ok := allowed[strings.ToUpper(key)]; ok { + environment = append(environment, entry) + } + } + return environment +} diff --git a/internal/store/lexical_helper_test.go b/internal/store/lexical_helper_test.go new file mode 100644 index 00000000..aeb36cbb --- /dev/null +++ b/internal/store/lexical_helper_test.go @@ -0,0 +1,152 @@ +package store + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestNewLexicalTokenizersDisabled(t *testing.T) { + tokenizers, err := newLexicalTokenizers(OpenOptions{}) + require.NoError(t, err) + require.Nil(t, tokenizers) +} + +func TestNewLexicalTokenizersArabicIsInProcess(t *testing.T) { + tokenizers, err := newLexicalTokenizers(OpenOptions{LexicalLanguages: []string{"ar"}}) + require.NoError(t, err) + require.Contains(t, tokenizers, "ar") + tokens, err := tokenizers["ar"].Tokenize(context.Background(), "والكتاب") + require.NoError(t, err) + require.Contains(t, tokens, "كتاب") +} + +func TestNewLexicalTokenizersRejectsUnknownLanguage(t *testing.T) { + _, err := newLexicalTokenizers(OpenOptions{LexicalLanguages: []string{"default"}}) + require.ErrorContains(t, err, "unsupported lexical language") +} + +func TestOpenWithOptionsLoadsKiwiHelperLazily(t *testing.T) { + ctx := context.Background() + s, err := OpenWithOptions(ctx, filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ + LexicalLanguages: []string{"ko"}, + LexicalKiwiCommand: "/definitely/missing/discrawl-kiwi", + LexicalKiwiModel: "/definitely/missing/kiwi-model", + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + err = s.UpsertMessage(ctx, MessageRecord{ + ID: "ko", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + }) + require.ErrorContains(t, err, "start ko lexical tokenizer") + require.NotContains(t, err.Error(), "Python") +} + +func lexicalHelperCommand(mode string) *exec.Cmd { + return &exec.Cmd{ + Path: os.Args[0], + Args: []string{os.Args[0], "-test.run=TestLexicalTokenizerHelperProcess", "--", mode}, + Env: append(os.Environ(), "DISCRAWL_LEXICAL_HELPER=1"), + } +} + +func TestLexicalTokenizerHelperProcess(t *testing.T) { + if os.Getenv("DISCRAWL_LEXICAL_HELPER") != "1" { + return + } + mode := os.Args[len(os.Args)-1] + switch mode { + case "startup-error": + fmt.Println(`{"error":"missing tokenizer package"}`) + os.Exit(2) + case "malformed-startup": + fmt.Println("not-json") + os.Exit(2) + case "stderr-startup": + fmt.Fprintln(os.Stderr, "tokenizer stderr") + os.Exit(2) + case "ready", "response-error", "malformed-response": + fmt.Println(`{"ready":true}`) + default: + fmt.Println(`{"ready":true}`) + } + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + var request map[string]string + if err := json.Unmarshal(scanner.Bytes(), &request); err != nil { + fmt.Println(`{"error":"bad request"}`) + continue + } + if mode == "response-error" { + fmt.Println(`{"error":"tokenization failed"}`) + continue + } + if mode == "malformed-response" { + fmt.Println("not-json") + continue + } + tokens := request["text"] + " tokenized" + if request["text"] == "오늘 저녁먹음 기록" { + tokens = "오늘 저녁 먹 음 기록" + } + response, err := json.Marshal(map[string]string{"tokens": tokens}) + if err != nil { + fmt.Println(`{"error":"encode response"}`) + continue + } + fmt.Println(string(response)) + } +} + +func TestNewLexicalTokenizersCreatesLazyHelpers(t *testing.T) { + tokenizers, err := newLexicalTokenizers(OpenOptions{ + LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, + LexicalKiwiCommand: "/definitely/missing/discrawl-kiwi", + LexicalJaCommand: "/definitely/missing/discrawl-ja", + LexicalZhCommand: "/definitely/missing/discrawl-zh", + }) + require.NoError(t, err) + require.ElementsMatch(t, []string{"ko", "ja", "zh", "ar"}, keys(tokenizers)) +} + +func keys(tokenizers map[string]LexicalTokenizer) []string { + out := make([]string, 0, len(tokenizers)) + for language := range tokenizers { + out = append(out, language) + } + return out +} + +func TestStartHelperLexicalTokenizerUsesReadyScript(t *testing.T) { + dir := t.TempDir() + script := filepath.Join(dir, "discrawl-ja") + require.NoError(t, os.WriteFile(script, []byte(`#!/bin/sh +echo '{"ready":true}' +read line +echo '{"tokens":"tokyo"}' +`), 0o700)) + tokenizer, err := startHelperLexicalTokenizer("ja", script, "discrawl-ja", nil) + require.NoError(t, err) + defer func() { _ = tokenizer.Close() }() + tokens, err := tokenizer.Tokenize(context.Background(), "東京") + require.NoError(t, err) + require.Equal(t, "tokyo", tokens) +} + +func TestOpenReadOnlyWithOptionsRejectsUnknownLanguage(t *testing.T) { + _, err := OpenReadOnlyWithOptions(context.Background(), filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ + LexicalLanguages: []string{"nope"}, + }) + require.ErrorContains(t, err, "unsupported lexical language") +} diff --git a/internal/store/lexical_install.go b/internal/store/lexical_install.go deleted file mode 100644 index fbaf375a..00000000 --- a/internal/store/lexical_install.go +++ /dev/null @@ -1,136 +0,0 @@ -package store - -import ( - "bytes" - "context" - "errors" - "fmt" - "os" - "os/exec" - "runtime" - "strings" -) - -type LexicalInstallResult struct { - Packages []string - Output string -} - -type lexicalCommandRunner func(context.Context, string, ...string) ([]byte, error) - -func InstallLexicalPackages( - ctx context.Context, - python string, - languages []string, -) (LexicalInstallResult, error) { - return installLexicalPackagesWithRunner(ctx, python, languages, runLexicalCommand) -} - -func installLexicalPackagesWithRunner( - ctx context.Context, - python string, - languages []string, - run lexicalCommandRunner, -) (LexicalInstallResult, error) { - packages, err := lexicalPythonPackages(languages) - if err != nil { - return LexicalInstallResult{}, err - } - if len(packages) == 0 { - if len(languages) == 0 { - return LexicalInstallResult{}, errors.New("no search.lexical languages are configured") - } - return LexicalInstallResult{}, nil - } - python, err = expandLexicalPython(python) - if err != nil { - return LexicalInstallResult{}, err - } - spec, err := pythonCommandSpec(python, runtime.GOOS == "windows") - if err != nil { - return LexicalInstallResult{}, err - } - probeArgs := append([]string{}, spec.ArgsPrefix...) - probeArgs = append(probeArgs, "-c", "import sys; print(sys.prefix != sys.base_prefix)") - output, err := run(ctx, spec.Path, probeArgs...) - if err != nil { - return LexicalInstallResult{}, fmt.Errorf("check lexical Python virtual environment: %w", err) - } - if !strings.EqualFold(strings.TrimSpace(string(output)), "true") { - return LexicalInstallResult{}, errors.New( - "lexical package installation requires a virtual environment; configure search.lexical.python to its interpreter", - ) - } - installArgs := append([]string{}, spec.ArgsPrefix...) - installArgs = append( - installArgs, - "-m", - "pip", - "install", - "--disable-pip-version-check", - "--no-input", - "--require-virtualenv", - ) - installArgs = append(installArgs, packages...) - output, err = run(ctx, spec.Path, installArgs...) - if err != nil { - return LexicalInstallResult{}, fmt.Errorf("install lexical Python packages: %w", err) - } - return LexicalInstallResult{ - Packages: packages, - Output: strings.TrimSpace(string(output)), - }, nil -} - -func lexicalPythonPackages(languages []string) ([]string, error) { - packages := make([]string, 0, len(languages)+1) - seen := make(map[string]struct{}, len(languages)) - for _, language := range languages { - if _, ok := seen[language]; ok { - continue - } - seen[language] = struct{}{} - switch language { - case "ko": - continue - case "ja": - packages = append(packages, "sudachipy==0.6.11", "sudachidict_core==20260723") - case "zh": - packages = append(packages, "jieba==0.42.1") - case "ar": - packages = append(packages, "snowballstemmer==3.1.1") - default: - return nil, fmt.Errorf("unsupported lexical language %q", language) - } - } - return packages, nil -} - -func runLexicalCommand(ctx context.Context, path string, args ...string) ([]byte, error) { - var output bytes.Buffer - cmd := &exec.Cmd{ - Path: path, - Args: append([]string{path}, args...), - Env: lexicalWorkerEnvironment(os.Environ()), - Stdout: &output, - Stderr: &output, - } - if err := cmd.Start(); err != nil { - return nil, err - } - wait := make(chan error, 1) - go func() { - wait <- cmd.Wait() - }() - select { - case err := <-wait: - if err != nil { - return output.Bytes(), fmt.Errorf("%w: %s", err, strings.TrimSpace(output.String())) - } - return output.Bytes(), nil - case <-ctx.Done(): - _ = cmd.Process.Kill() - <-wait - return output.Bytes(), ctx.Err() - } -} diff --git a/internal/store/lexical_kiwi.go b/internal/store/lexical_kiwi.go index 6f0cb1a4..c0dcf147 100644 --- a/internal/store/lexical_kiwi.go +++ b/internal/store/lexical_kiwi.go @@ -1,26 +1,10 @@ package store import ( - "bufio" - "context" - "encoding/json" - "errors" "fmt" - "io" - "os" "os/exec" - "path/filepath" - "strings" - "sync" - "time" ) -type kiwiLexicalResponse struct { - Ready bool `json:"ready,omitempty"` - Tokens string `json:"tokens,omitempty"` - Error string `json:"error,omitempty"` -} - func startKiwiLexicalTokenizer(command, model string) (LexicalTokenizer, error) { cmd, err := newKiwiLexicalCommand(command, model) if err != nil { @@ -30,182 +14,17 @@ func startKiwiLexicalTokenizer(command, model string) (LexicalTokenizer, error) } func startKiwiLexicalTokenizerCommand(cmd *exec.Cmd) (LexicalTokenizer, error) { - tokenizer := &externalLexicalTokenizer{ - language: "ko", - command: cmd, - } - stdin, err := cmd.StdinPipe() - if err != nil { - return nil, err - } - stdout, err := cmd.StdoutPipe() - if err != nil { - _ = stdin.Close() - return nil, err - } - cmd.Stderr = &tokenizer.stderr - tokenizer.stdin = stdin - tokenizer.stdout = bufio.NewScanner(stdout) - tokenizer.stdout.Buffer(make([]byte, 4096), 8*1024*1024) - if err := cmd.Start(); err != nil { - return nil, fmt.Errorf("start Korean Kiwi tokenizer: %w", err) - } - response, err := tokenizer.readStartup() - if err != nil { - _ = tokenizer.Close() - return nil, fmt.Errorf("initialize Korean Kiwi tokenizer: %w", err) - } - if !response.Ready { - _ = tokenizer.Close() - return nil, errors.New(response.Error) - } - return tokenizer, nil + return startHelperLexicalTokenizerCommand("ko", cmd) } func newKiwiLexicalCommand(command, model string) (*exec.Cmd, error) { - command, err := expandLexicalPath(command, "discrawl-kiwi") - if err != nil { - return nil, fmt.Errorf("resolve Kiwi helper: %w", err) - } - if !filepath.IsAbs(command) { - if command != "discrawl-kiwi" { - return nil, fmt.Errorf( - "unsupported Kiwi helper %q; use an absolute path or discrawl-kiwi", - command, - ) - } - command, err = exec.LookPath(command) - if err != nil { - return nil, fmt.Errorf("find Kiwi helper: %w", err) - } - } - model, err = expandLexicalPath(model, "") + model, err := expandLexicalPath(model, "") if err != nil { return nil, fmt.Errorf("resolve Kiwi model: %w", err) } - args := []string{command} + var extra []string if model != "" { - args = append(args, "--model", model) - } - return &exec.Cmd{ - Path: command, - Args: args, - Env: lexicalWorkerEnvironment(os.Environ()), - }, nil -} - -func expandLexicalPath(path, fallback string) (string, error) { - path = strings.TrimSpace(path) - if path == "" { - return fallback, nil - } - if !strings.HasPrefix(path, "~/") { - return path, nil - } - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(home, strings.TrimPrefix(path, "~/")), nil -} - -type externalLexicalTokenizer struct { - language string - command *exec.Cmd - stdin io.WriteCloser - stdout *bufio.Scanner - stderr lockedBuffer - mutex sync.Mutex -} - -func (t *externalLexicalTokenizer) readStartup() (kiwiLexicalResponse, error) { - result := make(chan struct { - response kiwiLexicalResponse - err error - }, 1) - go func() { - response, err := t.readResponse() - result <- struct { - response kiwiLexicalResponse - err error - }{response, err} - }() - select { - case output := <-result: - return output.response, output.err - case <-time.After(30 * time.Second): - _ = t.command.Process.Kill() - return kiwiLexicalResponse{}, errors.New("kiwi tokenizer startup timed out after 30s") - } -} - -func (t *externalLexicalTokenizer) Tokenize(ctx context.Context, text string) (string, error) { - t.mutex.Lock() - defer t.mutex.Unlock() - if err := ctx.Err(); err != nil { - return "", err - } - request, err := json.Marshal(map[string]string{"text": text}) - if err != nil { - return "", err - } - if _, err := t.stdin.Write(append(request, '\n')); err != nil { - return "", fmt.Errorf("write %s tokenizer request: %w", t.language, err) - } - result := make(chan struct { - response kiwiLexicalResponse - err error - }, 1) - go func() { - response, err := t.readResponse() - result <- struct { - response kiwiLexicalResponse - err error - }{response, err} - }() - select { - case output := <-result: - if output.err != nil { - return "", output.err - } - if output.response.Error != "" { - return "", errors.New(output.response.Error) - } - return output.response.Tokens, nil - case <-ctx.Done(): - _ = t.command.Process.Kill() - return "", ctx.Err() - case <-time.After(30 * time.Second): - _ = t.command.Process.Kill() - return "", errors.New("kiwi tokenizer response timed out after 30s") - } -} - -func (t *externalLexicalTokenizer) readResponse() (kiwiLexicalResponse, error) { - if !t.stdout.Scan() { - if err := t.stdout.Err(); err != nil { - return kiwiLexicalResponse{}, err - } - if detail := strings.TrimSpace(t.stderr.String()); detail != "" { - return kiwiLexicalResponse{}, errors.New(detail) - } - return kiwiLexicalResponse{}, io.EOF - } - var response kiwiLexicalResponse - if err := json.Unmarshal(t.stdout.Bytes(), &response); err != nil { - return kiwiLexicalResponse{}, fmt.Errorf("decode Kiwi tokenizer response: %w", err) - } - return response, nil -} - -func (t *externalLexicalTokenizer) Close() error { - if t == nil || t.command == nil || t.command.Process == nil { - return nil - } - _ = t.stdin.Close() - err := t.command.Wait() - if errors.Is(err, os.ErrProcessDone) { - return nil + extra = []string{"--model", model} } - return err + return newHelperLexicalCommand(command, "discrawl-kiwi", extra) } diff --git a/internal/store/lexical_python.go b/internal/store/lexical_python.go deleted file mode 100644 index 8ebae1af..00000000 --- a/internal/store/lexical_python.go +++ /dev/null @@ -1,327 +0,0 @@ -package store - -import ( - "bufio" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" - "sync" - "time" -) - -type OpenOptions struct { - LexicalLanguages []string - LexicalPython string - LexicalKiwiCommand string - LexicalKiwiModel string -} - -type pythonLexicalTokenizer struct { - language string - command *exec.Cmd - stdin io.WriteCloser - stdout *bufio.Scanner - stderr lockedBuffer - mutex sync.Mutex -} - -type lockedBuffer struct { - mutex sync.Mutex - buffer bytes.Buffer -} - -func (b *lockedBuffer) Write(data []byte) (int, error) { - b.mutex.Lock() - defer b.mutex.Unlock() - return b.buffer.Write(data) -} - -func (b *lockedBuffer) String() string { - b.mutex.Lock() - defer b.mutex.Unlock() - return b.buffer.String() -} - -type pythonLexicalResponse struct { - Ready bool `json:"ready,omitempty"` - Tokens string `json:"tokens,omitempty"` - Error string `json:"error,omitempty"` -} - -func OpenWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { - tokenizers, err := newLexicalTokenizers(opts) - if err != nil { - return nil, err - } - return openWithLexicalTokenizers(ctx, path, tokenizers) -} - -func OpenReadOnlyWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { - tokenizers, err := newLexicalTokenizers(opts) - if err != nil { - return nil, err - } - return openReadOnlyWithLexicalTokenizers(ctx, path, tokenizers) -} - -func newLexicalTokenizers(opts OpenOptions) (map[string]LexicalTokenizer, error) { - if len(opts.LexicalLanguages) == 0 { - return nil, nil - } - python, err := expandLexicalPython(opts.LexicalPython) - if err != nil { - return nil, err - } - tokenizers := make(map[string]LexicalTokenizer, len(opts.LexicalLanguages)) - for _, language := range opts.LexicalLanguages { - switch language { - case "ko": - command := opts.LexicalKiwiCommand - model := opts.LexicalKiwiModel - tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { - tokenizer, err := startKiwiLexicalTokenizer(command, model) - if err != nil { - return nil, fmt.Errorf("start ko lexical tokenizer: %w", err) - } - return tokenizer, nil - }) - default: - tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { - return startPythonLexicalTokenizer(python, language) - }) - } - } - return tokenizers, nil -} - -func expandLexicalPython(python string) (string, error) { - python = strings.TrimSpace(python) - if python == "" { - return "python3", nil - } - if !strings.HasPrefix(python, "~/") { - return python, nil - } - home, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("resolve lexical Python home directory: %w", err) - } - return filepath.Join(home, strings.TrimPrefix(python, "~/")), nil -} - -func startPythonLexicalTokenizer( - python string, - language string, -) (*pythonLexicalTokenizer, error) { - command, err := newPythonLexicalCommand(python, language) - if err != nil { - return nil, err - } - return startPythonLexicalTokenizerCommand(command, language) -} - -func startPythonLexicalTokenizerCommand( - command *exec.Cmd, - language string, -) (*pythonLexicalTokenizer, error) { - tokenizer := &pythonLexicalTokenizer{language: language, command: command} - stdin, err := command.StdinPipe() - if err != nil { - return nil, err - } - stdout, err := command.StdoutPipe() - if err != nil { - _ = stdin.Close() - return nil, err - } - command.Stderr = &tokenizer.stderr - tokenizer.stdin = stdin - tokenizer.stdout = bufio.NewScanner(stdout) - tokenizer.stdout.Buffer(make([]byte, 4096), 8*1024*1024) - if err := command.Start(); err != nil { - return nil, fmt.Errorf("start %s lexical tokenizer with %s: %w", language, command.Path, err) - } - response, err := tokenizer.readStartupResponse() - if err != nil { - _ = tokenizer.Close() - return nil, fmt.Errorf("initialize %s lexical tokenizer: %w", language, err) - } - if !response.Ready { - _ = tokenizer.Close() - return nil, fmt.Errorf("initialize %s lexical tokenizer: %s", language, response.Error) - } - return tokenizer, nil -} - -func (p *pythonLexicalTokenizer) readStartupResponse() (pythonLexicalResponse, error) { - type startupResult struct { - response pythonLexicalResponse - err error - } - result := make(chan startupResult, 1) - go func() { - response, err := p.readResponse() - result <- startupResult{response: response, err: err} - }() - select { - case startup := <-result: - return startup.response, startup.err - case <-time.After(30 * time.Second): - _ = p.command.Process.Kill() - return pythonLexicalResponse{}, errors.New("startup timed out after 30s") - } -} - -func (p *pythonLexicalTokenizer) Tokenize(ctx context.Context, text string) (string, error) { - p.mutex.Lock() - defer p.mutex.Unlock() - if err := ctx.Err(); err != nil { - return "", err - } - request, err := json.Marshal(map[string]string{"text": text}) - if err != nil { - return "", err - } - if _, err := p.stdin.Write(append(request, '\n')); err != nil { - return "", fmt.Errorf("write tokenizer request: %w", err) - } - response, err := p.readResponseContext(ctx) - if err != nil { - return "", err - } - if response.Error != "" { - return "", errors.New(response.Error) - } - return response.Tokens, nil -} - -func (p *pythonLexicalTokenizer) readResponseContext(ctx context.Context) (pythonLexicalResponse, error) { - readCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - type responseResult struct { - response pythonLexicalResponse - err error - } - result := make(chan responseResult, 1) - go func() { - response, err := p.readResponse() - result <- responseResult{response: response, err: err} - }() - select { - case response := <-result: - return response.response, response.err - case <-readCtx.Done(): - _ = p.command.Process.Kill() - return pythonLexicalResponse{}, fmt.Errorf("tokenizer response: %w", readCtx.Err()) - } -} - -func (p *pythonLexicalTokenizer) readResponse() (pythonLexicalResponse, error) { - if !p.stdout.Scan() { - err := p.stdout.Err() - if err == nil { - err = io.EOF - } - detail := strings.TrimSpace(p.stderr.String()) - if detail != "" { - return pythonLexicalResponse{}, fmt.Errorf("%w: %s", err, detail) - } - return pythonLexicalResponse{}, err - } - var response pythonLexicalResponse - if err := json.Unmarshal(p.stdout.Bytes(), &response); err != nil { - return pythonLexicalResponse{}, fmt.Errorf("decode tokenizer response: %w", err) - } - return response, nil -} - -func (p *pythonLexicalTokenizer) Close() error { - if p == nil || p.command == nil || p.command.Process == nil { - return nil - } - _ = p.stdin.Close() - err := p.command.Wait() - if errors.Is(err, os.ErrProcessDone) { - return nil - } - return err -} - -const pythonLexicalWorker = ` -import json -import re -import sys -import unicodedata - -language = sys.argv[1] - -try: - if language == "default": - engine = None - elif language == "ja": - from sudachipy import dictionary, tokenizer as sudachi_tokenizer - engine = dictionary.Dictionary().create() - split_mode = sudachi_tokenizer.Tokenizer.SplitMode.A - elif language == "zh": - import jieba - engine = jieba - elif language == "ar": - import snowballstemmer - engine = snowballstemmer.stemmer("arabic") - else: - raise RuntimeError("unsupported lexical language: " + language) -except Exception as error: - print(json.dumps({"ready": False, "error": str(error)}, ensure_ascii=False), flush=True) - raise SystemExit(2) - -print(json.dumps({"ready": True}, ensure_ascii=False), flush=True) - -def unique(tokens): - output = [] - seen = set() - for token in tokens: - token = unicodedata.normalize("NFKC", token).strip().lower() - if token and token not in seen: - seen.add(token) - output.append(token) - return output - -def tokenize(text): - if language == "default": - return unique(re.findall(r"[^\W\d_]+", text, flags=re.UNICODE)) - if language == "ja": - output = [] - for token in engine.tokenize(text, split_mode): - output.append(token.surface()) - base = token.dictionary_form() - if base != "*": - output.append(base) - return unique(output) - if language == "zh": - return unique(engine.cut_for_search(text)) - words = re.findall(r"[^\W\d_]+", text, flags=re.UNICODE) - normalized = [re.sub(r"[\u064b-\u065f\u0670\u0640]", "", word) for word in words] - variants = list(normalized) - for word in normalized: - if word.startswith(("وال", "فال", "بال", "كال", "لال")) and len(word) > 4: - variants.append(word[3:]) - elif word.startswith("ال") and len(word) > 3: - variants.append(word[2:]) - elif word[:1] in ("و", "ف", "ب", "ك", "ل") and len(word) > 3: - variants.append(word[1:]) - return unique(variants + engine.stemWords(variants)) - -for line in sys.stdin: - try: - request = json.loads(line) - print(json.dumps({"tokens": " ".join(tokenize(request.get("text", "")))}, ensure_ascii=False), flush=True) - except Exception as error: - print(json.dumps({"error": str(error)}, ensure_ascii=False), flush=True) -` diff --git a/internal/store/lexical_python_command.go b/internal/store/lexical_python_command.go deleted file mode 100644 index 1cf1d7cc..00000000 --- a/internal/store/lexical_python_command.go +++ /dev/null @@ -1,102 +0,0 @@ -package store - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" -) - -type lexicalCommandSpec struct { - Path string - ArgsPrefix []string -} - -func pythonCommandSpec(python string, windows bool) (lexicalCommandSpec, error) { - python = strings.TrimSpace(python) - if python == "" { - python = "python3" - } - if isAbsolutePythonPath(python, windows) { - return lexicalCommandSpec{Path: python}, nil - } - allowed := map[string][]string{ - "python": nil, - "python3": nil, - } - if windows { - allowed["python.exe"] = nil - allowed["python3.exe"] = nil - allowed["py"] = []string{"-3"} - allowed["py.exe"] = []string{"-3"} - } - prefix, ok := allowed[python] - if !ok { - return lexicalCommandSpec{}, fmt.Errorf( - "unsupported lexical Python interpreter %q; use an absolute path or python/python3", - python, - ) - } - path, err := exec.LookPath(python) - if err != nil { - return lexicalCommandSpec{}, fmt.Errorf("find lexical Python interpreter %q: %w", python, err) - } - return lexicalCommandSpec{Path: path, ArgsPrefix: prefix}, nil -} - -func newPythonLexicalCommand(python string, language string) (*exec.Cmd, error) { - spec, err := pythonCommandSpec(python, runtime.GOOS == "windows") - if err != nil { - return nil, err - } - args := append([]string{spec.Path}, spec.ArgsPrefix...) - args = append(args, "-u", "-c", pythonLexicalWorker, language) - return &exec.Cmd{ - Path: spec.Path, - Args: args, - Env: lexicalWorkerEnvironment(os.Environ()), - }, nil -} - -func isAbsolutePythonPath(path string, windows bool) bool { - if windows { - if strings.HasPrefix(path, `\\`) { - return true - } - return len(path) >= 3 && - ((path[0] >= 'A' && path[0] <= 'Z') || (path[0] >= 'a' && path[0] <= 'z')) && - path[1] == ':' && - (path[2] == '\\' || path[2] == '/') - } - return filepath.IsAbs(path) -} - -func lexicalWorkerEnvironment(parent []string) []string { - allowed := map[string]struct{}{ - "HOME": {}, - "LANG": {}, - "LC_ALL": {}, - "PATH": {}, - "PATHEXT": {}, - "SYSTEMROOT": {}, - "TEMP": {}, - "TMP": {}, - "TMPDIR": {}, - "VIRTUAL_ENV": {}, - "WINDIR": {}, - } - environment := make([]string, 0, len(allowed)+2) - for _, entry := range parent { - key, _, ok := strings.Cut(entry, "=") - if !ok { - continue - } - if _, ok := allowed[strings.ToUpper(key)]; ok { - environment = append(environment, entry) - } - } - environment = append(environment, "PYTHONNOUSERSITE=1", "PYTHONUTF8=1") - return environment -} diff --git a/internal/store/lexical_python_test.go b/internal/store/lexical_python_test.go deleted file mode 100644 index 895d7785..00000000 --- a/internal/store/lexical_python_test.go +++ /dev/null @@ -1,210 +0,0 @@ -package store - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestPythonLexicalTokenizerCommandProtocol(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizerCommand( - lexicalHelperCommand("ready"), - "test", - ) - require.NoError(t, err) - - tokens, err := tokenizer.Tokenize(context.Background(), "mixed text") - require.NoError(t, err) - require.Equal(t, "mixed text tokenized", tokens) - require.NoError(t, tokenizer.Close()) -} - -func TestLockedBuffer(t *testing.T) { - var buffer lockedBuffer - written, err := buffer.Write([]byte("tokenizer stderr")) - require.NoError(t, err) - require.Equal(t, len("tokenizer stderr"), written) - require.Equal(t, "tokenizer stderr", buffer.String()) -} - -func TestPythonLexicalTokenizerCommandStartupError(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizerCommand( - lexicalHelperCommand("startup-error"), - "test", - ) - require.Nil(t, tokenizer) - require.ErrorContains(t, err, "missing tokenizer package") -} - -func TestPythonLexicalTokenizerCommandMalformedStartup(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizerCommand( - lexicalHelperCommand("malformed-startup"), - "test", - ) - require.Nil(t, tokenizer) - require.ErrorContains(t, err, "decode tokenizer response") -} - -func TestPythonLexicalTokenizerCommandStartupStderr(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizerCommand( - lexicalHelperCommand("stderr-startup"), - "test", - ) - require.Nil(t, tokenizer) - require.ErrorContains(t, err, "tokenizer stderr") -} - -func TestPythonLexicalTokenizerCommandResponseError(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizerCommand( - lexicalHelperCommand("response-error"), - "test", - ) - require.NoError(t, err) - defer func() { _ = tokenizer.Close() }() - - _, err = tokenizer.Tokenize(context.Background(), "mixed text") - require.ErrorContains(t, err, "tokenization failed") -} - -func TestPythonLexicalTokenizerHonorsCanceledContext(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizerCommand( - lexicalHelperCommand("ready"), - "test", - ) - require.NoError(t, err) - defer func() { _ = tokenizer.Close() }() - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - _, err = tokenizer.Tokenize(ctx, "mixed text") - require.ErrorIs(t, err, context.Canceled) -} - -func TestPythonLexicalTokenizerReportsWriteAfterClose(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizerCommand( - lexicalHelperCommand("ready"), - "test", - ) - require.NoError(t, err) - require.NoError(t, tokenizer.Close()) - - _, err = tokenizer.Tokenize(context.Background(), "mixed text") - require.ErrorContains(t, err, "write tokenizer request") -} - -func TestNewLexicalTokenizersDisabled(t *testing.T) { - tokenizers, err := newLexicalTokenizers(OpenOptions{}) - require.NoError(t, err) - require.Nil(t, tokenizers) -} - -func TestPythonLexicalTokenizerDefaultWorker(t *testing.T) { - tokenizer, err := startPythonLexicalTokenizer("python3", "default") - require.NoError(t, err) - defer func() { _ = tokenizer.Close() }() - - tokens, err := tokenizer.Tokenize(context.Background(), "Mixed CASE 123") - require.NoError(t, err) - require.Equal(t, "mixed case", tokens) -} - -func TestNewLexicalTokenizersExpandsPythonHomePath(t *testing.T) { - python, err := exec.LookPath("python3") - require.NoError(t, err) - home := t.TempDir() - require.NoError(t, os.Symlink(python, filepath.Join(home, "python3"))) - t.Setenv("HOME", home) - - tokenizers, err := newLexicalTokenizers(OpenOptions{ - LexicalLanguages: []string{"default"}, - LexicalPython: "~/python3", - }) - require.NoError(t, err) - require.Contains(t, tokenizers, "default") - closeLexicalTokenizers(tokenizers) -} - -func TestOpenWithOptionsLoadsKiwiHelperLazily(t *testing.T) { - ctx := context.Background() - s, err := OpenWithOptions(ctx, filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ - LexicalLanguages: []string{"ko"}, - LexicalKiwiCommand: "/definitely/missing/discrawl-kiwi", - LexicalKiwiModel: "/definitely/missing/kiwi-model", - }) - require.NoError(t, err) - defer func() { _ = s.Close() }() - - err = s.UpsertMessage(ctx, MessageRecord{ - ID: "ko", GuildID: "g1", ChannelID: "c1", - CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), - Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, - }) - require.ErrorContains(t, err, "start ko lexical tokenizer") - require.NotContains(t, err.Error(), "Python") -} - -func lexicalHelperCommand(mode string) *exec.Cmd { - return &exec.Cmd{ - Path: os.Args[0], - Args: []string{os.Args[0], "-test.run=TestLexicalTokenizerHelperProcess", "--", mode}, - Env: append(os.Environ(), "DISCRAWL_LEXICAL_HELPER=1"), - } -} - -func TestLexicalTokenizerHelperProcess(t *testing.T) { - if os.Getenv("DISCRAWL_LEXICAL_HELPER") != "1" { - return - } - mode := os.Args[len(os.Args)-1] - if mode == "startup-error" { - fmt.Println(`{"error":"missing tokenizer package"}`) - os.Exit(2) - } - if mode == "malformed-startup" { - fmt.Println(`not-json`) - os.Exit(2) - } - if mode == "stderr-startup" { - _, _ = fmt.Fprintln(os.Stderr, "tokenizer stderr") - os.Exit(2) - } - fmt.Println(`{"ready":true}`) - scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - var request map[string]string - if err := json.Unmarshal(scanner.Bytes(), &request); err != nil { - fmt.Printf("{\"error\":%q}\n", err.Error()) - continue - } - if mode == "response-error" { - fmt.Println(`{"error":"tokenization failed"}`) - continue - } - if mode == "malformed-response" { - fmt.Println(`not-json`) - continue - } - response, err := json.Marshal(map[string]string{ - "tokens": func() string { - if request["text"] == "오늘 저녁먹음 기록" { - return "오늘 저녁 먹 음 기록" - } - return request["text"] + " tokenized" - }(), - }) - if err != nil { - fmt.Printf("{\"error\":%q}\n", err.Error()) - continue - } - fmt.Println(string(response)) - } - os.Exit(0) -} diff --git a/internal/store/lexical_review_test.go b/internal/store/lexical_review_test.go index e4906565..055317a0 100644 --- a/internal/store/lexical_review_test.go +++ b/internal/store/lexical_review_test.go @@ -2,162 +2,26 @@ package store import ( "context" - "errors" - "fmt" "os" "path/filepath" - "slices" "strings" "testing" "github.com/stretchr/testify/require" ) -func TestLexicalPythonPackagesSelectOnlyConfiguredLanguages(t *testing.T) { - packages, err := lexicalPythonPackages([]string{"ko", "zh"}) - require.NoError(t, err) - require.Equal(t, []string{ - "jieba==0.42.1", - }, packages) - require.NotContains(t, packages, "kiwipiepy==0.23.2") - require.NotContains(t, packages, "sudachipy==0.6.11") - require.NotContains(t, packages, "snowballstemmer==3.1.1") -} - -func TestLexicalPythonPackagesRejectsEmptyAndUnknownLanguages(t *testing.T) { - packages, err := lexicalPythonPackages(nil) - require.NoError(t, err) - require.Empty(t, packages) - - _, err = lexicalPythonPackages([]string{"ko", "unknown"}) - require.ErrorContains(t, err, `unsupported lexical language "unknown"`) -} - -func TestInstallLexicalPackagesRejectsNoConfiguredLanguages(t *testing.T) { - _, err := InstallLexicalPackages(context.Background(), "python3", nil) - require.ErrorContains(t, err, "no search.lexical languages") -} - -func TestInstallLexicalPackagesSkipsKiwiOnlyConfiguration(t *testing.T) { - result, err := InstallLexicalPackages(context.Background(), "python3", []string{"ko"}) - require.NoError(t, err) - require.Empty(t, result.Packages) -} - -func TestInstallLexicalPackagesRequiresVirtualEnvironment(t *testing.T) { - _, err := installLexicalPackagesWithRunner( - context.Background(), - "/tmp/python", - []string{"zh"}, - func(context.Context, string, ...string) ([]byte, error) { - return []byte("false\n"), nil - }, - ) - require.ErrorContains(t, err, "virtual environment") -} - -func TestInstallLexicalPackagesUsesPinnedSelectedPackages(t *testing.T) { - var commands [][]string - result, err := installLexicalPackagesWithRunner( - context.Background(), - "/tmp/python", - []string{"ja", "ar"}, - func(_ context.Context, path string, args ...string) ([]byte, error) { - commands = append(commands, append([]string{path}, args...)) - if slices.Contains(args, "import sys; print(sys.prefix != sys.base_prefix)") { - return []byte("true\n"), nil - } - return []byte("installed\n"), nil - }, - ) - require.NoError(t, err) - require.Equal(t, []string{ - "sudachipy==0.6.11", - "sudachidict_core==20260723", - "snowballstemmer==3.1.1", - }, result.Packages) - require.Len(t, commands, 2) - require.Equal(t, []string{ - "/tmp/python", - "-m", - "pip", - "install", - "--disable-pip-version-check", - "--no-input", - "--require-virtualenv", - "sudachipy==0.6.11", - "sudachidict_core==20260723", - "snowballstemmer==3.1.1", - }, commands[1]) -} - -func TestInstallLexicalPackagesReportsBoundaryFailures(t *testing.T) { - _, err := installLexicalPackagesWithRunner( - context.Background(), - "/tmp/python", - nil, - func(context.Context, string, ...string) ([]byte, error) { - return nil, errors.New("must not run") - }, - ) - require.ErrorContains(t, err, "no search.lexical languages") - - _, err = installLexicalPackagesWithRunner( - context.Background(), - "sh", - []string{"zh"}, - func(context.Context, string, ...string) ([]byte, error) { - return nil, errors.New("must not run") - }, - ) - require.ErrorContains(t, err, "unsupported lexical Python interpreter") - - _, err = installLexicalPackagesWithRunner( - context.Background(), - "/tmp/python", - []string{"zh"}, - func(context.Context, string, ...string) ([]byte, error) { - return nil, errors.New("probe failed") - }, - ) - require.ErrorContains(t, err, "check lexical Python virtual environment") - - calls := 0 - _, err = installLexicalPackagesWithRunner( - context.Background(), - "/tmp/python", - []string{"zh"}, - func(context.Context, string, ...string) ([]byte, error) { - calls++ - if calls == 1 { - return []byte("True\n"), nil - } - return []byte("pip failed"), errors.New("install failed") - }, - ) - require.ErrorContains(t, err, "install lexical Python packages") -} - -func TestPythonCommandSpecAllowsKnownLaunchersAndAbsolutePaths(t *testing.T) { - spec, err := pythonCommandSpec("python3", false) - require.NoError(t, err) - require.NotEmpty(t, spec.Path) - - spec, err = pythonCommandSpec("/opt/discrawl/tokenizers/bin/python", false) - require.NoError(t, err) - require.Equal(t, "/opt/discrawl/tokenizers/bin/python", spec.Path) - - spec, err = pythonCommandSpec(`C:\discrawl-tokenizers\Scripts\python.exe`, true) - require.NoError(t, err) - require.Equal(t, `C:\discrawl-tokenizers\Scripts\python.exe`, spec.Path) - - _, err = pythonCommandSpec("sh", false) - require.ErrorContains(t, err, "unsupported lexical Python interpreter") - - require.True(t, isAbsolutePythonPath(`\\server\share\python.exe`, true)) - require.True(t, isAbsolutePythonPath(`D:/venv/python.exe`, true)) - require.False(t, isAbsolutePythonPath(`venv\python.exe`, true)) - require.False(t, isAbsolutePythonPath("venv/python", false)) +func TestLexicalHelperStatusesDescribeRuntimes(t *testing.T) { + statuses := LexicalHelperStatuses(OpenOptions{ + LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, + LexicalKiwiCommand: "discrawl-kiwi", + LexicalJaCommand: "/opt/discrawl-ja", + }) + require.Equal(t, []LexicalHelperStatus{ + {Language: "ko", Runtime: "helper", Command: "discrawl-kiwi"}, + {Language: "ja", Runtime: "helper", Command: "/opt/discrawl-ja"}, + {Language: "zh", Runtime: "helper", Command: "discrawl-zh"}, + {Language: "ar", Runtime: "in-process"}, + }, statuses) } func TestLexicalWorkerEnvironmentDropsParentSecrets(t *testing.T) { @@ -174,8 +38,8 @@ func TestLexicalWorkerEnvironmentDropsParentSecrets(t *testing.T) { require.Contains(t, joined, "PATH=/usr/bin") require.Contains(t, joined, "HOME=/tmp/home") require.Contains(t, joined, "LANG=en_US.UTF-8") - require.Contains(t, joined, "VIRTUAL_ENV=/tmp/venv") - require.Contains(t, joined, "PYTHONNOUSERSITE=1") + require.NotContains(t, joined, "VIRTUAL_ENV") + require.NotContains(t, joined, "PYTHONNOUSERSITE") require.NotContains(t, joined, "DISCORD_BOT_TOKEN") require.NotContains(t, joined, "OPENAI_API_KEY") require.NotContains(t, joined, "PIP_INDEX_URL") @@ -198,17 +62,18 @@ func TestKiwiCommandUsesGoHelperAndConfiguredModel(t *testing.T) { }, command.Args) } -func TestKiwiCommandRejectsArbitraryRelativeCommand(t *testing.T) { +func TestHelperCommandsRejectArbitraryRelativeCommands(t *testing.T) { _, err := newKiwiLexicalCommand("sh", "/tmp/model") - require.ErrorContains(t, err, "unsupported Kiwi helper") + require.ErrorContains(t, err, "unsupported discrawl-kiwi helper") + _, err = newHelperLexicalCommand("sh", "discrawl-ja", nil) + require.ErrorContains(t, err, "unsupported discrawl-ja helper") + _, err = newHelperLexicalCommand("sh", "discrawl-zh", nil) + require.ErrorContains(t, err, "unsupported discrawl-zh helper") } func TestKiwiTokenizerCommandProtocol(t *testing.T) { - tokenizer, err := startKiwiLexicalTokenizerCommand( - lexicalHelperCommand("ready"), - ) + tokenizer, err := startKiwiLexicalTokenizerCommand(lexicalHelperCommand("ready")) require.NoError(t, err) - tokens, err := tokenizer.Tokenize(context.Background(), "오늘 저녁먹음 기록") require.NoError(t, err) require.Equal(t, "오늘 저녁 먹 음 기록", tokens) @@ -218,29 +83,21 @@ func TestKiwiTokenizerCommandProtocol(t *testing.T) { } func TestKiwiTokenizerCommandStartupFailures(t *testing.T) { - tokenizer, err := startKiwiLexicalTokenizerCommand( - lexicalHelperCommand("startup-error"), - ) + tokenizer, err := startKiwiLexicalTokenizerCommand(lexicalHelperCommand("startup-error")) require.Nil(t, tokenizer) require.ErrorContains(t, err, "missing tokenizer package") - tokenizer, err = startKiwiLexicalTokenizerCommand( - lexicalHelperCommand("malformed-startup"), - ) + tokenizer, err = startKiwiLexicalTokenizerCommand(lexicalHelperCommand("malformed-startup")) require.Nil(t, tokenizer) - require.ErrorContains(t, err, "decode Kiwi tokenizer response") + require.ErrorContains(t, err, "decode lexical tokenizer response") - tokenizer, err = startKiwiLexicalTokenizerCommand( - lexicalHelperCommand("stderr-startup"), - ) + tokenizer, err = startKiwiLexicalTokenizerCommand(lexicalHelperCommand("stderr-startup")) require.Nil(t, tokenizer) require.ErrorContains(t, err, "tokenizer stderr") } func TestKiwiTokenizerCommandResponseErrorAndCancellation(t *testing.T) { - tokenizer, err := startKiwiLexicalTokenizerCommand( - lexicalHelperCommand("response-error"), - ) + tokenizer, err := startKiwiLexicalTokenizerCommand(lexicalHelperCommand("response-error")) require.NoError(t, err) defer func() { _ = tokenizer.Close() }() @@ -254,14 +111,12 @@ func TestKiwiTokenizerCommandResponseErrorAndCancellation(t *testing.T) { } func TestKiwiTokenizerCommandMalformedResponse(t *testing.T) { - tokenizer, err := startKiwiLexicalTokenizerCommand( - lexicalHelperCommand("malformed-response"), - ) + tokenizer, err := startKiwiLexicalTokenizerCommand(lexicalHelperCommand("malformed-response")) require.NoError(t, err) defer func() { _ = tokenizer.Close() }() _, err = tokenizer.Tokenize(context.Background(), "text") - require.ErrorContains(t, err, "decode Kiwi tokenizer response") + require.ErrorContains(t, err, "decode lexical tokenizer response") var nilTokenizer *externalLexicalTokenizer require.NoError(t, nilTokenizer.Close()) @@ -278,69 +133,3 @@ func TestKiwiCommandDefaultHelperAndOptionalModel(t *testing.T) { require.Equal(t, helper, command.Path) require.Equal(t, []string{helper}, command.Args) } - -func TestRunLexicalCommandCapturesOutputAndFailure(t *testing.T) { - t.Setenv("DISCRAWL_INSTALL_HELPER", "1") - output, err := runLexicalCommand( - context.Background(), - os.Args[0], - "-test.run=TestLexicalInstallCommandHelperProcess", - "--", - "success", - ) - require.NoError(t, err) - require.Contains(t, string(output), "installed\n") - - output, err = runLexicalCommand( - context.Background(), - os.Args[0], - "-test.run=TestLexicalInstallCommandHelperProcess", - "--", - "failure", - ) - require.ErrorContains(t, err, "exit status") - require.Contains(t, string(output), "install failed") - - _, err = runLexicalCommand(context.Background(), "/definitely/missing/discrawl-command") - require.Error(t, err) -} - -func TestRunLexicalCommandDropsParentSecrets(t *testing.T) { - t.Setenv("DISCORD_BOT_TOKEN", "discord-secret") - t.Setenv("OPENAI_API_KEY", "openai-secret") - t.Setenv("PIP_INDEX_URL", "https://user:password@example.invalid/simple") - output, err := runLexicalCommand( - context.Background(), - os.Args[0], - "-test.run=TestLexicalInstallCommandHelperProcess", - "--", - "environment", - ) - require.NoError(t, err) - require.Contains(t, string(output), "clean\n") -} - -func TestLexicalInstallCommandHelperProcess(t *testing.T) { - if !slices.Contains(os.Args, "-test.run=TestLexicalInstallCommandHelperProcess") { - return - } - switch os.Args[len(os.Args)-1] { - case "success": - fmt.Println("installed") - os.Exit(0) - case "failure": - fmt.Println("install failed") - os.Exit(2) - case "environment": - for _, key := range []string{"DISCORD_BOT_TOKEN", "OPENAI_API_KEY", "PIP_INDEX_URL"} { - if os.Getenv(key) != "" { - fmt.Println(key) - os.Exit(4) - } - } - fmt.Println("clean") - os.Exit(0) - default: - os.Exit(3) - } -} diff --git a/internal/store/lexical_status.go b/internal/store/lexical_status.go new file mode 100644 index 00000000..d70b53b0 --- /dev/null +++ b/internal/store/lexical_status.go @@ -0,0 +1,53 @@ +package store + +import "strings" + +type LexicalHelperStatus struct { + Language string `json:"language"` + Runtime string `json:"runtime"` + Command string `json:"command,omitempty"` +} + +func LexicalHelperStatuses(opts OpenOptions) []LexicalHelperStatus { + statuses := make([]LexicalHelperStatus, 0, len(opts.LexicalLanguages)) + for _, language := range opts.LexicalLanguages { + switch language { + case "ko": + command := strings.TrimSpace(opts.LexicalKiwiCommand) + if command == "" { + command = "discrawl-kiwi" + } + statuses = append(statuses, LexicalHelperStatus{ + Language: language, + Runtime: "helper", + Command: command, + }) + case "ja": + command := strings.TrimSpace(opts.LexicalJaCommand) + if command == "" { + command = "discrawl-ja" + } + statuses = append(statuses, LexicalHelperStatus{ + Language: language, + Runtime: "helper", + Command: command, + }) + case "zh": + command := strings.TrimSpace(opts.LexicalZhCommand) + if command == "" { + command = "discrawl-zh" + } + statuses = append(statuses, LexicalHelperStatus{ + Language: language, + Runtime: "helper", + Command: command, + }) + case "ar": + statuses = append(statuses, LexicalHelperStatus{ + Language: language, + Runtime: "in-process", + }) + } + } + return statuses +} diff --git a/internal/store/lexical_tokenizers.go b/internal/store/lexical_tokenizers.go new file mode 100644 index 00000000..d209560b --- /dev/null +++ b/internal/store/lexical_tokenizers.go @@ -0,0 +1,74 @@ +package store + +import ( + "context" + "fmt" +) + +type OpenOptions struct { + LexicalLanguages []string + LexicalKiwiCommand string + LexicalKiwiModel string + LexicalJaCommand string + LexicalZhCommand string +} + +func OpenWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { + tokenizers, err := newLexicalTokenizers(opts) + if err != nil { + return nil, err + } + return openWithLexicalTokenizers(ctx, path, tokenizers) +} + +func OpenReadOnlyWithOptions(ctx context.Context, path string, opts OpenOptions) (*Store, error) { + tokenizers, err := newLexicalTokenizers(opts) + if err != nil { + return nil, err + } + return openReadOnlyWithLexicalTokenizers(ctx, path, tokenizers) +} + +func newLexicalTokenizers(opts OpenOptions) (map[string]LexicalTokenizer, error) { + if len(opts.LexicalLanguages) == 0 { + return nil, nil + } + tokenizers := make(map[string]LexicalTokenizer, len(opts.LexicalLanguages)) + for _, language := range opts.LexicalLanguages { + switch language { + case "ko": + command := opts.LexicalKiwiCommand + model := opts.LexicalKiwiModel + tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + tokenizer, err := startKiwiLexicalTokenizer(command, model) + if err != nil { + return nil, fmt.Errorf("start ko lexical tokenizer: %w", err) + } + return tokenizer, nil + }) + case "ja": + command := opts.LexicalJaCommand + tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + tokenizer, err := startHelperLexicalTokenizer("ja", command, "discrawl-ja", nil) + if err != nil { + return nil, fmt.Errorf("start ja lexical tokenizer: %w", err) + } + return tokenizer, nil + }) + case "zh": + command := opts.LexicalZhCommand + tokenizers[language] = newLazyLexicalTokenizer(func() (LexicalTokenizer, error) { + tokenizer, err := startHelperLexicalTokenizer("zh", command, "discrawl-zh", nil) + if err != nil { + return nil, fmt.Errorf("start zh lexical tokenizer: %w", err) + } + return tokenizer, nil + }) + case "ar": + tokenizers[language] = newArabicLexicalTokenizer() + default: + return nil, fmt.Errorf("unsupported lexical language %q", language) + } + } + return tokenizers, nil +} diff --git a/internal/store/multilingual_benchmark_test.go b/internal/store/multilingual_benchmark_test.go index 1fb6173d..82b85bb2 100644 --- a/internal/store/multilingual_benchmark_test.go +++ b/internal/store/multilingual_benchmark_test.go @@ -28,9 +28,10 @@ func TestMultilingualLexicalQualityBenchmark(t *testing.T) { defer func() { _ = baseline.Close() }() multilingual, err := OpenWithOptions(ctx, filepath.Join(root, "multilingual.db"), OpenOptions{ LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, - LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), LexicalKiwiCommand: os.Getenv("DISCRAWL_KIWI_HELPER"), LexicalKiwiModel: os.Getenv("DISCRAWL_KIWI_MODEL"), + LexicalJaCommand: os.Getenv("DISCRAWL_JA_HELPER"), + LexicalZhCommand: os.Getenv("DISCRAWL_ZH_HELPER"), }) require.NoError(t, err) defer func() { _ = multilingual.Close() }() diff --git a/internal/store/multilingual_e2e_test.go b/internal/store/multilingual_e2e_test.go index 9a270b40..d478a33f 100644 --- a/internal/store/multilingual_e2e_test.go +++ b/internal/store/multilingual_e2e_test.go @@ -17,9 +17,10 @@ func TestMultilingualLexicalSearchE2E(t *testing.T) { ctx := context.Background() s, err := OpenWithOptions(ctx, filepath.Join(t.TempDir(), "discrawl.db"), OpenOptions{ LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, - LexicalPython: os.Getenv("DISCRAWL_TOKENIZER_PYTHON"), LexicalKiwiCommand: os.Getenv("DISCRAWL_KIWI_HELPER"), LexicalKiwiModel: os.Getenv("DISCRAWL_KIWI_MODEL"), + LexicalJaCommand: os.Getenv("DISCRAWL_JA_HELPER"), + LexicalZhCommand: os.Getenv("DISCRAWL_ZH_HELPER"), }) require.NoError(t, err) defer func() { _ = s.Close() }() diff --git a/tools/discrawl-ja/README.md b/tools/discrawl-ja/README.md new file mode 100644 index 00000000..a281d082 --- /dev/null +++ b/tools/discrawl-ja/README.md @@ -0,0 +1,14 @@ +# discrawl-ja + +Optional Japanese lexical helper. It uses the pure-Go +[`github.com/ikawaha/kagome/v2`](https://pkg.go.dev/github.com/ikawaha/kagome/v2) +tokenizer in Search mode with the embedded MeCab-IPADIC dictionary. + +This binary is not linked into the default Discrawl release. Build it only when +Japanese lexical fields are enabled. + +```bash +go build -o discrawl-ja . +``` + +The helper speaks the same newline-delimited JSON protocol as `discrawl-kiwi`. diff --git a/tools/discrawl-ja/go.mod b/tools/discrawl-ja/go.mod new file mode 100644 index 00000000..4766c3e6 --- /dev/null +++ b/tools/discrawl-ja/go.mod @@ -0,0 +1,10 @@ +module github.com/openclaw/discrawl/tools/discrawl-ja + +go 1.26 + +require ( + github.com/ikawaha/kagome-dict/ipa v1.2.6 + github.com/ikawaha/kagome/v2 v2.11.0 +) + +require github.com/ikawaha/kagome-dict v1.1.7 // indirect diff --git a/tools/discrawl-ja/go.sum b/tools/discrawl-ja/go.sum new file mode 100644 index 00000000..709824ae --- /dev/null +++ b/tools/discrawl-ja/go.sum @@ -0,0 +1,6 @@ +github.com/ikawaha/kagome-dict v1.1.7 h1:O/uAL+WCGhp6kT0+szxBSPaSM4i+vdArSefFvJE4Nug= +github.com/ikawaha/kagome-dict v1.1.7/go.mod h1:9tvk7/jZkvYt40foxkB9CqSAAknoQrIPfzqQd05UkFw= +github.com/ikawaha/kagome-dict/ipa v1.2.6 h1:Bcvm4jgxAAnTIKb6ckqUKBiFDN0wuanFfycMuYt7xGQ= +github.com/ikawaha/kagome-dict/ipa v1.2.6/go.mod h1:ONdTMUAKMCq9yx4s69QRtPcJLEMVM0BNNYQrMCJLWb0= +github.com/ikawaha/kagome/v2 v2.11.0 h1:R914EkRzay9qtUbsFzEbcdZ3wHwwSPvbPkuBI1oIf78= +github.com/ikawaha/kagome/v2 v2.11.0/go.mod h1:6mYPezBou+iNVnX9uNa00Sfu6S6t2zcM8Nv1EW9Y9so= diff --git a/tools/discrawl-ja/main.go b/tools/discrawl-ja/main.go new file mode 100644 index 00000000..fc37738f --- /dev/null +++ b/tools/discrawl-ja/main.go @@ -0,0 +1,81 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/ikawaha/kagome-dict/ipa" + "github.com/ikawaha/kagome/v2/tokenizer" +) + +type request struct { + Text string `json:"text"` +} + +type response struct { + Ready bool `json:"ready,omitempty"` + Tokens string `json:"tokens,omitempty"` + Error string `json:"error,omitempty"` + Version string `json:"version,omitempty"` +} + +func main() { + analyzer, err := tokenizer.New(ipa.Dict(), tokenizer.OmitBosEos()) + if err != nil { + writeResponse(response{Error: err.Error()}) + os.Exit(2) + } + writeResponse(response{Ready: true, Version: "kagome-ipa-search"}) + scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(make([]byte, 4096), 8*1024*1024) + for scanner.Scan() { + var input request + if err := json.Unmarshal(scanner.Bytes(), &input); err != nil { + writeResponse(response{Error: fmt.Sprintf("decode request: %v", err)}) + continue + } + writeResponse(response{Tokens: strings.Join(tokenize(analyzer, input.Text), " ")}) + } + if err := scanner.Err(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func tokenize(analyzer *tokenizer.Tokenizer, text string) []string { + seen := make(map[string]struct{}) + var tokens []string + add := func(form string) { + form = strings.ToLower(strings.TrimSpace(form)) + if form == "" || form == "*" { + return + } + if _, ok := seen[form]; ok { + return + } + seen[form] = struct{}{} + tokens = append(tokens, form) + } + for _, token := range analyzer.Analyze(text, tokenizer.Search) { + add(token.Surface) + if base := tokenBaseForm(token); base != "" { + add(base) + } + } + return tokens +} + +func tokenBaseForm(token tokenizer.Token) string { + features := token.Features() + if len(features) > 6 { + return features[6] + } + return "" +} + +func writeResponse(output response) { + _ = json.NewEncoder(os.Stdout).Encode(output) +} diff --git a/tools/discrawl-ja/main_test.go b/tools/discrawl-ja/main_test.go new file mode 100644 index 00000000..fd7dc724 --- /dev/null +++ b/tools/discrawl-ja/main_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "testing" + + "github.com/ikawaha/kagome-dict/ipa" + "github.com/ikawaha/kagome/v2/tokenizer" +) + +func TestTokenizeSearchSplitsCompounds(t *testing.T) { + analyzer, err := tokenizer.New(ipa.Dict(), tokenizer.OmitBosEos()) + if err != nil { + t.Fatal(err) + } + tokens := tokenize(analyzer, "東京都庁に行きます") + found := false + for _, token := range tokens { + if token == "東京" { + found = true + break + } + } + if !found { + t.Fatalf("expected 東京 in %v", tokens) + } +} diff --git a/tools/discrawl-zh/README.md b/tools/discrawl-zh/README.md new file mode 100644 index 00000000..8cd8b019 --- /dev/null +++ b/tools/discrawl-zh/README.md @@ -0,0 +1,14 @@ +# discrawl-zh + +Optional Chinese lexical helper. It uses the pure-Go +[`github.com/go-ego/gse`](https://pkg.go.dev/github.com/go-ego/gse) +search-mode segmenter with the embedded default dictionary. + +This binary is not linked into the default Discrawl release. Build it only when +Chinese lexical fields are enabled. + +```bash +go build -o discrawl-zh . +``` + +The helper speaks the same newline-delimited JSON protocol as `discrawl-kiwi`. diff --git a/tools/discrawl-zh/go.mod b/tools/discrawl-zh/go.mod new file mode 100644 index 00000000..5aab131a --- /dev/null +++ b/tools/discrawl-zh/go.mod @@ -0,0 +1,7 @@ +module github.com/openclaw/discrawl/tools/discrawl-zh + +go 1.26 + +require github.com/go-ego/gse v1.0.2 + +require github.com/vcaesar/cedar v0.30.0 // indirect diff --git a/tools/discrawl-zh/go.sum b/tools/discrawl-zh/go.sum new file mode 100644 index 00000000..b28ed36b --- /dev/null +++ b/tools/discrawl-zh/go.sum @@ -0,0 +1,6 @@ +github.com/go-ego/gse v1.0.2 h1:+27lYFPhQEhA9igtdOsJPRKYL/k3TwYsxBF5jr6KFv4= +github.com/go-ego/gse v1.0.2/go.mod h1:Fy35G+q7VV7Et1zIKO8o/sW1kkugV3znXap/lF/11zc= +github.com/vcaesar/cedar v0.30.0 h1:9fSDpM7FTjjUdPiBUUa0MWYMRGSEcqgFXvppZcZ4d7Y= +github.com/vcaesar/cedar v0.30.0/go.mod h1:lyuGvALuZZDPNXwpzv/9LyxW+8Y6faN7zauFezNsnik= +github.com/vcaesar/tt v0.20.1 h1:D/jUeeVCNbq3ad8M7hhtB3J9x5RZ6I1n1eZ0BJp7M+4= +github.com/vcaesar/tt v0.20.1/go.mod h1:cH2+AwGAJm19Wa6xvEa+0r+sXDJBT0QgNQey6mwqLeU= diff --git a/tools/discrawl-zh/main.go b/tools/discrawl-zh/main.go new file mode 100644 index 00000000..392e56b3 --- /dev/null +++ b/tools/discrawl-zh/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/go-ego/gse" +) + +type request struct { + Text string `json:"text"` +} + +type response struct { + Ready bool `json:"ready,omitempty"` + Tokens string `json:"tokens,omitempty"` + Error string `json:"error,omitempty"` + Version string `json:"version,omitempty"` +} + +func main() { + var seg gse.Segmenter + if err := seg.LoadDictEmbed(); err != nil { + writeResponse(response{Error: err.Error()}) + os.Exit(2) + } + writeResponse(response{Ready: true, Version: "gse-search"}) + scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(make([]byte, 4096), 8*1024*1024) + for scanner.Scan() { + var input request + if err := json.Unmarshal(scanner.Bytes(), &input); err != nil { + writeResponse(response{Error: fmt.Sprintf("decode request: %v", err)}) + continue + } + writeResponse(response{Tokens: strings.Join(unique(seg.CutSearch(input.Text, true)), " ")}) + } + if err := scanner.Err(); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func unique(forms []string) []string { + seen := make(map[string]struct{}, len(forms)) + tokens := make([]string, 0, len(forms)) + for _, form := range forms { + form = strings.ToLower(strings.TrimSpace(form)) + if form == "" { + continue + } + if _, ok := seen[form]; ok { + continue + } + seen[form] = struct{}{} + tokens = append(tokens, form) + } + return tokens +} + +func writeResponse(output response) { + _ = json.NewEncoder(os.Stdout).Encode(output) +} diff --git a/tools/discrawl-zh/main_test.go b/tools/discrawl-zh/main_test.go new file mode 100644 index 00000000..c0099eb5 --- /dev/null +++ b/tools/discrawl-zh/main_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "testing" + + "github.com/go-ego/gse" +) + +func TestCutSearchKeepsCompoundParts(t *testing.T) { + var seg gse.Segmenter + if err := seg.LoadDictEmbed(); err != nil { + t.Fatal(err) + } + tokens := unique(seg.CutSearch("自然语言处理很有趣", true)) + found := false + for _, token := range tokens { + if token == "语言" { + found = true + break + } + } + if !found { + t.Fatalf("expected 语言 in %v", tokens) + } +} From 7f3f7387e95dbf1ae3662cd85c85a2bd18ffb41c Mon Sep 17 00:00:00 2001 From: "Jeffrey (Dongkyu) Kim" Date: Thu, 20 Aug 2026 01:36:06 +0900 Subject: [PATCH 5/5] fix(search): purge disabled lexical indexes --- README.md | 1 - docs/README.md | 2 +- docs/commands/lexical.md | 32 ------- docs/configuration.md | 2 +- docs/guides/search-modes.md | 4 +- internal/cli/cli.go | 6 -- internal/cli/lexical_commands.go | 31 ------ internal/cli/lexical_commands_test.go | 94 ------------------- internal/cli/output.go | 7 -- internal/store/lexical.go | 13 ++- internal/store/lexical_review_test.go | 14 --- internal/store/lexical_status.go | 53 ----------- internal/store/multilingual_lifecycle_test.go | 61 ++++++++++++ internal/store/write.go | 26 +++-- tools/discrawl-kiwi/install-kiwi.sh | 14 ++- 15 files changed, 105 insertions(+), 255 deletions(-) delete mode 100644 docs/commands/lexical.md delete mode 100644 internal/cli/lexical_commands.go delete mode 100644 internal/cli/lexical_commands_test.go delete mode 100644 internal/store/lexical_status.go diff --git a/README.md b/README.md index 8319ea2b..756f8f33 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,6 @@ The full documentation lives at **[discrawl.sh](https://discrawl.sh/)**: - [Command reference](docs/README.md) - [Sync sources](docs/guides/sync-sources.md) - [Search modes](docs/guides/search-modes.md) -- [Optional lexical tokenizer installation](docs/commands/lexical.md) - [Multilingual lexical benchmark](docs/benchmarks/multilingual-lexical.md) - [Git snapshot workflows](docs/guides/git-snapshots.md) - [Configuration](docs/configuration.md) diff --git a/docs/README.md b/docs/README.md index 98ee5941..b443cd04 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,7 +15,7 @@ Mirror Discord guilds into local SQLite. Search server history without depending ## Pick your path - **New here?** Read [Install](install.html) and run `discrawl init`. -- **Already have a bot?** Jump to [`sync`](commands/sync.html), [`search`](commands/search.html), and optional [`lexical install`](commands/lexical.html). +- **Already have a bot?** Jump to [`sync`](commands/sync.html) and [`search`](commands/search.html). - **Just want to read a shared archive?** Use [`subscribe`](commands/subscribe.html) for Git snapshots, or [`subscribe-cloud`](commands/subscribe-cloud.html) for a Worker-fronted archive - no Discord token needed. - **Need DM search?** [`wiretap`](commands/wiretap.html) imports local Discord Desktop cache. - **Want semantic search?** Configure [Embeddings](guides/embeddings.html), then run [`embed`](commands/embed.html). diff --git a/docs/commands/lexical.md b/docs/commands/lexical.md deleted file mode 100644 index 0198cd87..00000000 --- a/docs/commands/lexical.md +++ /dev/null @@ -1,32 +0,0 @@ -# `lexical` - -Shows the Go helpers required by `search.lexical.languages`. Discrawl never -downloads or installs tokenizer packages. - -## Usage - -```bash -discrawl lexical install -``` - -| Language | Runtime | Command | -| --- | --- | --- | -| `ko` | helper | `discrawl-kiwi` (`github.com/codingpot/kiwigo` + Kiwi 0.23.2) | -| `ja` | helper | `discrawl-ja` (`github.com/ikawaha/kagome/v2` Search) | -| `zh` | helper | `discrawl-zh` (`github.com/go-ego/gse` CutSearch) | -| `ar` | in-process | none | - -Tokenizer helpers start lazily only when an enabled language is first used for -indexing or search. Opening an archive never starts a helper. - -```toml -[search.lexical] -languages = ["ko", "ja", "zh", "ar"] -kiwi_command = "~/.local/share/discrawl/bin/discrawl-kiwi" -kiwi_model = "~/.local/share/discrawl/models/kiwi/base" -ja_command = "~/.local/share/discrawl/bin/discrawl-ja" -zh_command = "~/.local/share/discrawl/bin/discrawl-zh" -``` - -Helper sources live under `tools/discrawl-kiwi`, `tools/discrawl-ja`, and -`tools/discrawl-zh`. None of those helpers use Python. diff --git a/docs/configuration.md b/docs/configuration.md index 52884fdc..4e3712f1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -149,7 +149,7 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time - `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi through `github.com/codingpot/kiwigo`), Japanese (`ja`, Kagome Search through `discrawl-ja`), Chinese (`zh`, GSE CutSearch through `discrawl-zh`), and Arabic (`ar`, in-process light stemming). - Korean, Japanese, and Chinese use separately built Go helpers so the default Discrawl binary stays small and CGO-free. Arabic is implemented in-process. -- `discrawl lexical install` reports the required helpers. It never downloads packages. +- Optional lexical helpers are installed and built separately; Discrawl never downloads packages. - Tokenizer helpers load lazily on first indexing or search use. Commands that only inspect metadata do not start helpers. - Each enabled language adds an independent FTS5 table. Index and query text pass through the same tokenizer, and results from the default plus language-specific tables are merged with reciprocal rank fusion. - After adding or changing `search.lexical.languages`, run a writer command such as `discrawl sync` once so the configured lexical tables are built. Read-only commands never mutate the archive; new and edited messages update the tables automatically during later syncs. diff --git a/docs/guides/search-modes.md b/docs/guides/search-modes.md index 93bbc155..c338f984 100644 --- a/docs/guides/search-modes.md +++ b/docs/guides/search-modes.md @@ -49,8 +49,8 @@ ja_command = "~/.local/share/discrawl/bin/discrawl-ja" zh_command = "~/.local/share/discrawl/bin/discrawl-zh" ``` -`discrawl lexical install` reports those helper paths. It does not download -packages. +Discrawl does not download or install helper packages; build the optional +helpers separately and configure their absolute paths. Every message is analyzed into each configured field. This deliberately avoids language detection, so mixed-language Discord messages remain searchable diff --git a/internal/cli/cli.go b/internal/cli/cli.go index f2d0e87f..5eb4d0d5 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -128,7 +128,6 @@ var discrawlCommandSpecs = []discrawlCommandSpec{ {name: "cache-import", description: "Import Discord Desktop cache data (wiretap alias)."}, {name: "wiretap", description: "Import Discord Desktop cache data."}, {name: "search", description: "Search archived messages."}, - {name: "lexical", description: "Show configured Go lexical helpers."}, {name: "tui", description: "Explore the archive in an interactive terminal UI."}, {name: "messages", description: "List archived messages."}, {name: "digest", description: "Summarize recent archive activity."}, @@ -359,11 +358,6 @@ func (r *runtime) dispatch(rest []string) error { } autoShareUpdate := !hasBoolFlag(rest[1:], "--dm") return r.withLocalStoreRead(autoShareUpdate, func() error { return r.runSearch(rest[1:]) }) - case "lexical": - if hasHelpFlag(rest[1:]) { - return printCommandUsage(r.stdout, []string{"lexical"}) - } - return r.withConfig(func() error { return r.runLexical(rest[1:]) }) case "tui": if hasHelpArg(rest[1:]) { return r.runTUI(rest[1:]) diff --git a/internal/cli/lexical_commands.go b/internal/cli/lexical_commands.go deleted file mode 100644 index 9aa603a5..00000000 --- a/internal/cli/lexical_commands.go +++ /dev/null @@ -1,31 +0,0 @@ -package cli - -import ( - "errors" - - "github.com/openclaw/discrawl/internal/store" -) - -type lexicalInstallOutput struct { - Languages []string `json:"languages"` - Helpers []store.LexicalHelperStatus `json:"helpers"` -} - -func (r *runtime) runLexical(args []string) error { - if len(args) != 1 || args[0] != "install" { - return usageErr(errors.New("usage: discrawl lexical install")) - } - if len(r.cfg.Search.Lexical.Languages) == 0 { - return configErr(errors.New("search.lexical.languages is empty")) - } - return r.print(lexicalInstallOutput{ - Languages: append([]string(nil), r.cfg.Search.Lexical.Languages...), - Helpers: store.LexicalHelperStatuses(store.OpenOptions{ - LexicalLanguages: r.cfg.Search.Lexical.Languages, - LexicalKiwiCommand: r.cfg.Search.Lexical.KiwiCommand, - LexicalKiwiModel: r.cfg.Search.Lexical.KiwiModel, - LexicalJaCommand: r.cfg.Search.Lexical.JaCommand, - LexicalZhCommand: r.cfg.Search.Lexical.ZhCommand, - }), - }) -} diff --git a/internal/cli/lexical_commands_test.go b/internal/cli/lexical_commands_test.go deleted file mode 100644 index d66cfe46..00000000 --- a/internal/cli/lexical_commands_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package cli - -import ( - "bytes" - "context" - "os" - "path/filepath" - "testing" - - "github.com/openclaw/discrawl/internal/config" - "github.com/stretchr/testify/require" -) - -func TestRunLexicalInstallReportsConfiguredHelpers(t *testing.T) { - var stdout bytes.Buffer - r := &runtime{ - ctx: context.Background(), - cfg: config.Default(), - stdout: &stdout, - } - r.cfg.Search.Lexical.Languages = []string{"ko", "zh"} - r.cfg.Search.Lexical.ZhCommand = "/tmp/discrawl-zh" - - require.NoError(t, r.runLexical([]string{"install"})) - require.Contains(t, stdout.String(), "discrawl-kiwi") - require.Contains(t, stdout.String(), "/tmp/discrawl-zh") - require.NotContains(t, stdout.String(), "python") -} - -func TestRunLexicalInstallRequiresConfiguredLanguages(t *testing.T) { - r := &runtime{ - ctx: context.Background(), - cfg: config.Default(), - } - err := r.runLexical([]string{"install"}) - require.ErrorContains(t, err, "search.lexical.languages is empty") -} - -func TestRunLexicalInstallReportsUsage(t *testing.T) { - r := &runtime{ - ctx: context.Background(), - cfg: config.Default(), - } - r.cfg.Search.Lexical.Languages = []string{"ko"} - err := r.runLexical([]string{"unknown"}) - require.ErrorContains(t, err, "usage: discrawl lexical install") -} - -func TestRunLexicalInstallJSONOutput(t *testing.T) { - var stdout bytes.Buffer - r := &runtime{ - ctx: context.Background(), - cfg: config.Default(), - stdout: &stdout, - json: true, - } - r.cfg.Search.Lexical.Languages = []string{"ko", "ar"} - - require.NoError(t, r.runLexical([]string{"install"})) - require.JSONEq(t, `{ - "languages": ["ko", "ar"], - "helpers": [ - {"language":"ko","runtime":"helper","command":"discrawl-kiwi"}, - {"language":"ar","runtime":"in-process"} - ] - }`, stdout.String()) -} - -func TestLexicalHelp(t *testing.T) { - var stdout bytes.Buffer - require.NoError(t, Run(context.Background(), []string{"help", "lexical"}, &stdout, &bytes.Buffer{})) - require.Contains(t, stdout.String(), "discrawl lexical install") - require.Contains(t, stdout.String(), "discrawl-ja") - require.NotContains(t, stdout.String(), "Python packages") -} - -func TestRunDispatchesLexicalInstall(t *testing.T) { - configPath := filepath.Join(t.TempDir(), "config.toml") - require.NoError(t, os.WriteFile(configPath, []byte(` -version = 1 - -[discord] -token_source = "env" -`), 0o600)) - t.Setenv("DISCORD_BOT_TOKEN", "dummy") - - err := Run( - context.Background(), - []string{"--config", configPath, "lexical", "install"}, - &bytes.Buffer{}, - &bytes.Buffer{}, - ) - require.ErrorContains(t, err, "search.lexical.languages is empty") -} diff --git a/internal/cli/output.go b/internal/cli/output.go index 04f78978..427ba23f 100644 --- a/internal/cli/output.go +++ b/internal/cli/output.go @@ -248,13 +248,6 @@ Flags: --dm Search local desktop DM cache. --guild ID Restrict to one guild id. --guilds ID,ID Restrict to guild ids. -`, - "lexical": `Usage: - discrawl lexical install - -Show the Go helpers required by search.lexical.languages. -Korean uses discrawl-kiwi, Japanese uses discrawl-ja, Chinese uses discrawl-zh, and Arabic is in-process. -Discrawl never downloads tokenizers. `, "attachments": `Usage: discrawl attachments [flags] diff --git a/internal/store/lexical.go b/internal/store/lexical.go index f6103829..500145bf 100644 --- a/internal/store/lexical.go +++ b/internal/store/lexical.go @@ -148,8 +148,17 @@ func (s *Store) invalidateDisabledLexicalVersions(ctx context.Context) error { if err != nil { return err } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer rollback(tx) for _, scope := range disabledScopes { - if _, err := s.db.ExecContext( + language := knownScopes[scope] + if _, err := tx.ExecContext(ctx, "drop table if exists "+lexicalFTSTable(language)); err != nil { + return fmt.Errorf("drop disabled %s lexical index: %w", language, err) + } + if _, err := tx.ExecContext( ctx, `delete from sync_state where scope = ?`, scope, @@ -157,7 +166,7 @@ func (s *Store) invalidateDisabledLexicalVersions(ctx context.Context) error { return fmt.Errorf("invalidate disabled lexical index %s: %w", scope, err) } } - return nil + return tx.Commit() } func (s *Store) rebuildLexicalIndexes(ctx context.Context) error { diff --git a/internal/store/lexical_review_test.go b/internal/store/lexical_review_test.go index 055317a0..4761c814 100644 --- a/internal/store/lexical_review_test.go +++ b/internal/store/lexical_review_test.go @@ -10,20 +10,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestLexicalHelperStatusesDescribeRuntimes(t *testing.T) { - statuses := LexicalHelperStatuses(OpenOptions{ - LexicalLanguages: []string{"ko", "ja", "zh", "ar"}, - LexicalKiwiCommand: "discrawl-kiwi", - LexicalJaCommand: "/opt/discrawl-ja", - }) - require.Equal(t, []LexicalHelperStatus{ - {Language: "ko", Runtime: "helper", Command: "discrawl-kiwi"}, - {Language: "ja", Runtime: "helper", Command: "/opt/discrawl-ja"}, - {Language: "zh", Runtime: "helper", Command: "discrawl-zh"}, - {Language: "ar", Runtime: "in-process"}, - }, statuses) -} - func TestLexicalWorkerEnvironmentDropsParentSecrets(t *testing.T) { environment := lexicalWorkerEnvironment([]string{ "PATH=/usr/bin", diff --git a/internal/store/lexical_status.go b/internal/store/lexical_status.go deleted file mode 100644 index d70b53b0..00000000 --- a/internal/store/lexical_status.go +++ /dev/null @@ -1,53 +0,0 @@ -package store - -import "strings" - -type LexicalHelperStatus struct { - Language string `json:"language"` - Runtime string `json:"runtime"` - Command string `json:"command,omitempty"` -} - -func LexicalHelperStatuses(opts OpenOptions) []LexicalHelperStatus { - statuses := make([]LexicalHelperStatus, 0, len(opts.LexicalLanguages)) - for _, language := range opts.LexicalLanguages { - switch language { - case "ko": - command := strings.TrimSpace(opts.LexicalKiwiCommand) - if command == "" { - command = "discrawl-kiwi" - } - statuses = append(statuses, LexicalHelperStatus{ - Language: language, - Runtime: "helper", - Command: command, - }) - case "ja": - command := strings.TrimSpace(opts.LexicalJaCommand) - if command == "" { - command = "discrawl-ja" - } - statuses = append(statuses, LexicalHelperStatus{ - Language: language, - Runtime: "helper", - Command: command, - }) - case "zh": - command := strings.TrimSpace(opts.LexicalZhCommand) - if command == "" { - command = "discrawl-zh" - } - statuses = append(statuses, LexicalHelperStatus{ - Language: language, - Runtime: "helper", - Command: command, - }) - case "ar": - statuses = append(statuses, LexicalHelperStatus{ - Language: language, - Runtime: "in-process", - }) - } - } - return statuses -} diff --git a/internal/store/multilingual_lifecycle_test.go b/internal/store/multilingual_lifecycle_test.go index d0861bc3..d02f0248 100644 --- a/internal/store/multilingual_lifecycle_test.go +++ b/internal/store/multilingual_lifecycle_test.go @@ -70,6 +70,35 @@ func TestMultilingualIndexesTrackBatchDeletesAndGuildPurge(t *testing.T) { require.Empty(t, results) } +func TestDisabledLexicalIndexDoesNotRetainPurgedGuildData(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "discrawl.db") + enabled, err := openWithLexicalTokenizers(ctx, path, map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "회의기록": "회의 기록", + })}, + }) + require.NoError(t, err) + require.NoError(t, enabled.UpsertMessage(ctx, MessageRecord{ + ID: "message", GuildID: "guild", ChannelID: "channel", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "회의기록", NormalizedContent: "회의기록", RawJSON: `{}`, + })) + require.NoError(t, enabled.Close()) + + disabled, err := openWithLexicalTokenizers(ctx, path, nil) + require.NoError(t, err) + defer func() { _ = disabled.Close() }() + require.NoError(t, disabled.DeleteGuildData(ctx, "guild")) + + var tables int + require.NoError(t, disabled.DB().QueryRowContext( + ctx, + `select count(*) from sqlite_schema where type = 'table' and name = 'message_fts_ko'`, + ).Scan(&tables)) + require.Zero(t, tables) +} + func TestMultilingualIndexVersionSurvivesReopen(t *testing.T) { ctx := context.Background() path := filepath.Join(t.TempDir(), "discrawl.db") @@ -209,3 +238,35 @@ func TestMultilingualTokenizerFailureAbortsWrite(t *testing.T) { require.NoError(t, s.DB().QueryRowContext(ctx, `select count(*) from messages`).Scan(&count)) require.Zero(t, count) } + +func TestMultilingualDeleteUpsertDoesNotRequireTokenizer(t *testing.T) { + ctx := context.Background() + s, err := openWithLexicalTokenizers(ctx, filepath.Join(t.TempDir(), "discrawl.db"), map[string]LexicalTokenizer{ + "ko": stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "저녁먹음": "저녁 먹 음", + })}, + }) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + message := MessageRecord{ + ID: "ko", GuildID: "g1", ChannelID: "c1", + CreatedAt: time.Now().UTC().Format(time.RFC3339Nano), + Content: "저녁먹음", NormalizedContent: "저녁먹음", RawJSON: `{}`, + } + require.NoError(t, s.UpsertMessage(ctx, message)) + s.lexicalTokenizers["ko"] = failingLexicalTokenizer{err: errors.New("tokenizer unavailable")} + + message.DeletedAt = time.Now().UTC().Add(time.Minute).Format(time.RFC3339Nano) + require.NoError(t, s.UpsertMessage(ctx, message)) + var lexicalRows int + require.NoError(t, s.DB().QueryRowContext(ctx, `select count(*) from message_fts_ko`).Scan(&lexicalRows)) + require.Zero(t, lexicalRows) + s.lexicalTokenizers["ko"] = stubLexicalTokenizer{tokenize: replaceLexicalTerms(map[string]string{ + "저녁먹음": "저녁 먹 음", + })} + + results, err := s.SearchMessages(ctx, SearchOptions{Query: "저녁", Limit: 10}) + require.NoError(t, err) + require.Empty(t, results) +} diff --git a/internal/store/write.go b/internal/store/write.go index f5de542b..64e52569 100644 --- a/internal/store/write.go +++ b/internal/store/write.go @@ -266,9 +266,13 @@ func (s *Store) UpsertMessage(ctx context.Context, message MessageRecord) error } func (s *Store) UpsertMessageWithOptions(ctx context.Context, message MessageRecord, opts WriteOptions) error { - tokenized, err := s.tokenizeLexical(ctx, message.NormalizedContent) - if err != nil { - return err + var tokenized map[string]string + if message.DeletedAt == "" { + var err error + tokenized, err = s.tokenizeLexical(ctx, message.NormalizedContent) + if err != nil { + return err + } } tx, err := s.db.BeginTx(ctx, nil) if err != nil { @@ -287,10 +291,12 @@ func (s *Store) UpsertMessages(ctx context.Context, messages []MessageMutation) } tokenized := make([]map[string]string, len(messages)) for i, message := range messages { - var err error - tokenized[i], err = s.tokenizeLexical(ctx, message.Record.NormalizedContent) - if err != nil { - return err + if message.Record.DeletedAt == "" { + var err error + tokenized[i], err = s.tokenizeLexical(ctx, message.Record.NormalizedContent) + if err != nil { + return err + } } } tx, err := s.db.BeginTx(ctx, nil) @@ -372,6 +378,9 @@ func (s *Store) upsertMessageTx( if _, err := tx.ExecContext(ctx, deleteMessageFTSByRowIDSQL, rowID); err != nil { return err } + if err := s.upsertLexicalMessageTx(ctx, tx, message, tokenized); err != nil { + return err + } if message.DeletedAt != "" { if err := qtx.DeleteMessageEmbeddingsByMessage(ctx, message.ID); err != nil { return err @@ -387,9 +396,6 @@ func (s *Store) upsertMessageTx( `, rowID, message.ID, message.GuildID, message.ChannelID, nullable(message.AuthorID), message.AuthorName, message.ChannelName, message.NormalizedContent); err != nil { return err } - if err := s.upsertLexicalMessageTx(ctx, tx, message, tokenized); err != nil { - return err - } } queueEmbedding := opts.EnqueueEmbedding && (errors.Is(previousErr, sql.ErrNoRows) || previousNormalized.String != message.NormalizedContent || !jobExists) if queueEmbedding { diff --git a/tools/discrawl-kiwi/install-kiwi.sh b/tools/discrawl-kiwi/install-kiwi.sh index bc3eb21e..d19010aa 100644 --- a/tools/discrawl-kiwi/install-kiwi.sh +++ b/tools/discrawl-kiwi/install-kiwi.sh @@ -9,7 +9,7 @@ case "$(uname -s)" in esac case "$(uname -m)" in arm64|aarch64) - if [[ "$platform" == Linux ]]; then + if [[ "$(uname -s)" == Linux ]]; then architecture=aarch64 else architecture=arm64 @@ -25,6 +25,18 @@ work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT curl --fail --location "$url" --output "$work/kiwi.tgz" +case "$archive" in + kiwi_lnx_aarch64_v0.23.2.tgz) expected_sha256=7e093121a367087d21e7c696bcc69a505935b07798d1e95c87f3b66a646c124e ;; + kiwi_lnx_x86_64_v0.23.2.tgz) expected_sha256=0b6694a795891de22fb14ae46825403af02063450126282c18448d6562b97174 ;; + kiwi_mac_arm64_v0.23.2.tgz) expected_sha256=ac124e32e013e2089cb4d842e2b735a1e6b4f3b126cdf692d78fda1130b8a382 ;; + kiwi_mac_x86_64_v0.23.2.tgz) expected_sha256=422c4284cc73a7499a714090e4d2f1c039dbc565aa2b425e5a0c0656d7b483a5 ;; + *) echo "missing pinned Kiwi checksum for $archive" >&2; exit 1 ;; +esac +actual_sha256="$(shasum -a 256 "$work/kiwi.tgz" | awk '{print $1}')" +if [[ "$actual_sha256" != "$expected_sha256" ]]; then + echo "Kiwi archive checksum mismatch for $archive" >&2 + exit 1 +fi tar -xzf "$work/kiwi.tgz" -C "$work" sudo cp -R "$work/include/kiwi" /usr/local/include/ sudo cp -P "$work"/lib/libkiwi* /usr/local/lib/