diff --git a/docs/commands/sync.md b/docs/commands/sync.md index db81086f..15a367a5 100644 --- a/docs/commands/sync.md +++ b/docs/commands/sync.md @@ -43,7 +43,7 @@ discrawl sync --with-media | Command | Use when | Behavior | | --- | --- | --- | -| `discrawl sync` | routine refresh | skips member refreshes, checks live top-level channels plus active threads, only fetches new messages for channels with a stored cursor | +| `discrawl sync` | routine refresh | skips member refreshes, checks live top-level channels plus active threads, fetches one newest page when no cursor exists, and otherwise fetches only new messages | | `discrawl sync --update=auto` | hybrid Git/live refresh | applies the configured stale snapshot update mode first, then runs the routine live refresh | | `discrawl sync --update=force` | intentional exact reconciliation | replaces public snapshot tables first, then runs the routine live refresh | | `discrawl sync --all-channels` | repair pass | broad incremental sweep across every stored channel/thread, including archived threads | diff --git a/docs/guides/sync-sources.md b/docs/guides/sync-sources.md index f451bbec..27599a09 100644 --- a/docs/guides/sync-sources.md +++ b/docs/guides/sync-sources.md @@ -18,7 +18,7 @@ Sync modes control the Discord bot API side of a run. When `wiretap` is selected | Command | Use when | Behavior | | --- | --- | --- | -| `discrawl sync` | routine refresh | skips member refreshes, checks live top-level channels plus active threads, only fetches new messages for channels with a stored latest cursor | +| `discrawl sync` | routine refresh | skips member refreshes, checks live top-level channels plus active threads, fetches one newest page when no cursor exists, and otherwise fetches only new messages | | `discrawl sync --update=auto` | hybrid Git/live refresh | imports a stale Git snapshot first, usually as a changed-shard delta, then runs the routine live refresh | | `discrawl sync --all-channels` | repair pass | broad incremental sweep across every stored channel/thread, including archived threads | | `discrawl sync --full` | historical backfill | crawls older history until channels are complete; can take a long time on large servers | diff --git a/internal/syncer/message_sync.go b/internal/syncer/message_sync.go index 2571b59f..84ee178f 100644 --- a/internal/syncer/message_sync.go +++ b/internal/syncer/message_sync.go @@ -233,11 +233,16 @@ func (s *Syncer) syncChannelMessages(ctx context.Context, guildID string, channe } return s.syncFullChannelHistory(ctx, channel, state, embeddings, since, progress) } - if latestOnly && state.Latest == "" { + if shouldSkipChannelSync(channel, state) { return 0, nil } - if shouldSkipChannelSync(channel, state) || (latestOnly && shouldSkipLatestOnlyChannelSync(channel, state)) { - return 0, nil + if latestOnly { + if state.Latest == "" { + return s.syncLatestChannelHistory(ctx, channel, embeddings, since, progress) + } + if shouldSkipLatestOnlyChannelSync(channel, state) { + return 0, nil + } } return s.syncIncrementalChannelHistory(ctx, channel, state, embeddings, since, progress) } @@ -331,7 +336,7 @@ func (s *Syncer) syncFullChannelHistory(ctx context.Context, channel *discordgo. if before == "" && state.Latest != "" { before = state.Latest } - count, latest, err := s.syncBackfillPages(ctx, channel, before, newest, channel.Name, embeddings, since, progress) + count, latest, err := s.syncBackfillPages(ctx, channel, before, newest, channel.Name, embeddings, since, 0, progress) messageCount += count newest = maxSnowflake(newest, latest) if err != nil { @@ -352,6 +357,17 @@ func (s *Syncer) syncFullChannelHistory(ctx context.Context, channel *discordgo. return messageCount, nil } +func (s *Syncer) syncLatestChannelHistory(ctx context.Context, channel *discordgo.Channel, embeddings bool, since time.Time, progress *messageSyncProgress) (int, error) { + count, newest, err := s.syncBackfillPages(ctx, channel, "", "", channel.Name, embeddings, since, 1, progress) + if err != nil || newest == "" { + return count, err + } + if err := s.advanceChannelLatest(ctx, channel.ID, newest); err != nil { + return count, err + } + return count, nil +} + func (s *Syncer) syncIncrementalChannelHistory(ctx context.Context, channel *discordgo.Channel, state channelSyncState, embeddings bool, since time.Time, progress *messageSyncProgress) (int, error) { if state.Latest == "" { return s.bootstrapChannelHistory(ctx, channel, embeddings, since, progress) @@ -445,9 +461,10 @@ func (s *Syncer) syncForwardPages(ctx context.Context, channel *discordgo.Channe return messageCount, newest, nil } -func (s *Syncer) syncBackfillPages(ctx context.Context, channel *discordgo.Channel, before, latestFloor, channelName string, embeddings bool, since time.Time, progress *messageSyncProgress) (int, string, error) { +func (s *Syncer) syncBackfillPages(ctx context.Context, channel *discordgo.Channel, before, latestFloor, channelName string, embeddings bool, since time.Time, pageLimit int, progress *messageSyncProgress) (int, string, error) { messageCount := 0 newest := "" + pages := 0 for { page, err := s.client.ChannelMessages(ctx, channel.ID, 100, before, "") if err != nil { @@ -465,6 +482,7 @@ func (s *Syncer) syncBackfillPages(ctx context.Context, channel *discordgo.Chann return messageCount, newest, err } progress.touch(channel, len(eligible)) + pages++ newest = maxSnowflake(newest, pageNewest) messageCount += len(eligible) // Backfill pages are older than any previously synced head, so only @@ -496,6 +514,9 @@ func (s *Syncer) syncBackfillPages(ctx context.Context, channel *discordgo.Chann } break } + if pageLimit > 0 && pages >= pageLimit { + break + } } return messageCount, newest, nil } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 0520449e..d88b9c4c 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -3,6 +3,7 @@ package syncer import ( "context" "errors" + "fmt" "path/filepath" "sync" "testing" @@ -589,7 +590,7 @@ func TestSyncSkipMembersFlagSkipsMemberRefresh(t *testing.T) { require.Zero(t, client.memberCalls) } -func TestSyncLatestOnlySkipsChannelsWithoutLatestCursor(t *testing.T) { +func TestSyncLatestOnlyBootstrapsNewestPageWithoutCompletingHistory(t *testing.T) { t.Parallel() ctx := context.Background() @@ -597,33 +598,65 @@ func TestSyncLatestOnlySkipsChannelsWithoutLatestCursor(t *testing.T) { require.NoError(t, err) defer func() { _ = s.Close() }() + messages := make([]*discordgo.Message, 0, 101) + for id := 200; id >= 100; id-- { + messages = append(messages, &discordgo.Message{ + ID: fmt.Sprintf("%03d", id), + GuildID: "g1", + ChannelID: "thread", + Content: fmt.Sprintf("message %d", id), + Timestamp: time.Now().UTC(), + Author: &discordgo.User{ID: "u1", Username: "user"}, + }) + } client := &fakeClient{ guilds: []*discordgo.UserGuild{{ID: "g1", Name: "Guild"}}, guildByID: map[string]*discordgo.Guild{ "g1": {ID: "g1", Name: "Guild"}, }, channels: map[string][]*discordgo.Channel{ - "g1": { - {ID: "c1", GuildID: "g1", Name: "empty", Type: discordgo.ChannelTypeGuildText}, - }, + "g1": {{ID: "forum", GuildID: "g1", Name: "development-areas", Type: discordgo.ChannelTypeGuildForum}}, }, - messages: map[string][]*discordgo.Message{ - "c1": {{ - ID: "100", - GuildID: "g1", - ChannelID: "c1", - Content: "would bootstrap without latest-only", - Timestamp: time.Now().UTC(), - Author: &discordgo.User{ID: "u1", Username: "user"}, + guildThreads: map[string][]*discordgo.Channel{ + "g1": {{ + ID: "thread", + GuildID: "g1", + ParentID: "forum", + Name: "Beta Feedback", + Type: discordgo.ChannelTypeGuildPublicThread, + LastMessageID: "200", }}, }, + messages: map[string][]*discordgo.Message{"thread": messages}, } svc := New(client, s, nil) - stats, err := svc.Sync(ctx, SyncOptions{LatestOnly: true}) + stats, err := svc.Sync(ctx, SyncOptions{LatestOnly: true, SkipMembers: true}) require.NoError(t, err) - require.Zero(t, stats.Messages) - require.Zero(t, client.messageCalls["c1"]) + require.Equal(t, 100, stats.Messages) + require.Equal(t, 1, client.messageCalls["thread"]) + + oldest, newest, err := s.ChannelMessageBounds(ctx, "thread") + require.NoError(t, err) + require.Equal(t, "101", oldest) + require.Equal(t, "200", newest) + backfill, err := s.GetSyncState(ctx, channelBackfillScope("thread")) + require.NoError(t, err) + require.Equal(t, "101", backfill) + complete, err := s.GetSyncState(ctx, channelHistoryCompleteScope("thread")) + require.NoError(t, err) + require.Empty(t, complete) + + stats, err = svc.Sync(ctx, SyncOptions{Full: true, SkipMembers: true}) + require.NoError(t, err) + require.Equal(t, 1, stats.Messages) + oldest, newest, err = s.ChannelMessageBounds(ctx, "thread") + require.NoError(t, err) + require.Equal(t, "100", oldest) + require.Equal(t, "200", newest) + complete, err = s.GetSyncState(ctx, channelHistoryCompleteScope("thread")) + require.NoError(t, err) + require.Equal(t, "1", complete) } func TestSyncLatestOnlySkipsUnchangedIncompleteChannel(t *testing.T) {