diff --git a/android/app/build.gradle.kts b/android/app/build.gradle.kts index 2cb00e5f..b77c7eb7 100644 --- a/android/app/build.gradle.kts +++ b/android/app/build.gradle.kts @@ -107,6 +107,16 @@ android { // that before each assembleDebug / assembleRelease. sourceSets["main"].jniLibs.srcDirs("src/main/jniLibs") + // assets/fronting-groups/curated.json is generated into build/ by + // the syncFrontingGroupsAssets task (defined further down). Keeping + // generated output under build/ rather than src/ means stale copies + // can't outlive the canonical file in the source tree, and the + // standard build/ gitignore covers it without a carve-out under + // src/main/assets/. + sourceSets["main"].assets.srcDir( + layout.buildDirectory.dir("generated/curatedAssets") + ) + packaging { resources.excludes += setOf( "META-INF/AL2.0", @@ -142,6 +152,15 @@ dependencies { debugImplementation("androidx.compose.ui:ui-tooling") debugImplementation("androidx.compose.ui:ui-test-manifest") + + // Local JVM unit tests (`gradlew :app:test`). JUnit 4 plus the real + // org.json:json classes — by default android.jar's stubbed + // JSONObject methods all return null in unit tests, which makes + // ConfigStore round-trip tests untestable. The org.json artifact + // overrides those stubs in the test classpath without affecting + // the device runtime. + testImplementation("junit:junit:4.13.2") + testImplementation("org.json:json:20240303") } // -------------------------------------------------------------------------- @@ -222,3 +241,34 @@ tasks.configureEach { "mergeReleaseJniLibFolders" -> dependsOn("cargoBuildRelease") } } + +// -------------------------------------------------------------------------- +// Bundle assets/fronting-groups/curated.json into the APK so the Android +// UI's "Load curated fronting groups" button can read it without a network +// hop. The Rust crate is the single source of truth; we copy into a +// build/generated/ directory that is wired into sourceSets.main.assets +// above, so stale outputs can't survive the canonical file being deleted +// or renamed (a fresh `gradlew clean` wipes them) and we don't need a +// gitignore carve-out under src/main/assets/. +// -------------------------------------------------------------------------- +val syncFrontingGroupsAssets = + tasks.register("syncFrontingGroupsAssets") { + from(rustCrateDir.resolve("assets/fronting-groups")) + include("curated.json") + // Sub-folder so the asset opens at "fronting-groups/curated.json" + // (matches CuratedGroups.ASSET_PATH); without the sub-dir Android + // would expose it at the asset namespace root. + into(layout.buildDirectory.dir("generated/curatedAssets/fronting-groups")) + } + +tasks.configureEach { + when (name) { + // Asset merge runs before resource processing — depending on + // mergeDebugAssets / mergeReleaseAssets is the most precise + // hook, but preBuild also covers the lint/compile paths that + // need the file present (lintDebug, etc.). + "preBuild" -> dependsOn(syncFrontingGroupsAssets) + "mergeDebugAssets", + "mergeReleaseAssets" -> dependsOn(syncFrontingGroupsAssets) + } +} diff --git a/android/app/src/main/java/com/therealaleph/mhrv/ConfigStore.kt b/android/app/src/main/java/com/therealaleph/mhrv/ConfigStore.kt index 0b982737..b1cee822 100644 --- a/android/app/src/main/java/com/therealaleph/mhrv/ConfigStore.kt +++ b/android/app/src/main/java/com/therealaleph/mhrv/ConfigStore.kt @@ -77,6 +77,21 @@ enum class UiLang { AUTO, FA, EN } */ enum class Mode { APPS_SCRIPT, DIRECT, FULL } +/** + * One multi-edge fronting group. Mirrors the Rust `FrontingGroup` + * struct in `src/config.rs` and the desktop UI's round-tripped form. + * + * `domains` matches case-insensitively, exact OR dot-anchored suffix + * (`vercel.com` covers `*.vercel.com`). First group whose member + * matches wins, so put more-specific groups earlier in the list. + */ +data class FrontingGroup( + val name: String, + val ip: String, + val sni: String, + val domains: List, +) + data class MhrvConfig( val mode: Mode = Mode.APPS_SCRIPT, @@ -161,6 +176,16 @@ data class MhrvConfig( /** UI language toggle. Non-Rust; honoured only by the Android wrapper. */ val uiLang: UiLang = UiLang.AUTO, + + /** + * Multi-edge fronting groups (Vercel, Fastly, AWS CloudFront, …). + * Until v1.9.x the Android Save path silently dropped this field + * because it wasn't modelled here; round-tripping fixes that and + * unlocks the curated bundle loader. There's no in-app editor for + * the entries — users either load the curated bundle or import a + * config that contains them. See `assets/fronting-groups/curated.json`. + */ + val frontingGroups: List = emptyList(), ) { /** * Extract just the deployment ID from either a full @@ -279,6 +304,19 @@ data class MhrvConfig( UiLang.FA -> "fa" UiLang.EN -> "en" }) + + if (frontingGroups.isNotEmpty()) { + put("fronting_groups", JSONArray().apply { + for (g in frontingGroups) { + put(JSONObject().apply { + put("name", g.name) + put("ip", g.ip) + put("sni", g.sni) + put("domains", JSONArray().apply { g.domains.forEach { put(it) } }) + }) + } + }) + } } return obj.toString(2) } @@ -356,6 +394,18 @@ object ConfigStore { if (cleanBypassDohHosts.isNotEmpty()) { obj.put("bypass_doh_hosts", JSONArray().apply { cleanBypassDohHosts.forEach { put(it) } }) } + if (cfg.frontingGroups.isNotEmpty()) { + obj.put("fronting_groups", JSONArray().apply { + for (g in cfg.frontingGroups) { + put(JSONObject().apply { + put("name", g.name) + put("ip", g.ip) + put("sni", g.sni) + put("domains", JSONArray().apply { g.domains.forEach { put(it) } }) + }) + } + }) + } // Compress with DEFLATE then base64. val jsonBytes = obj.toString().toByteArray(Charsets.UTF_8) @@ -415,8 +465,13 @@ object ConfigStore { return false } - /** Parse config from a JSON object — shared by load() and decode(). */ - private fun loadFromJson(obj: JSONObject): MhrvConfig { + /** + * Parse config from a JSON object — shared by [load] and [decode]. + * `internal` rather than `private` so the JVM unit tests in + * `src/test/` can drive a JSON-only round-trip without going + * through the disk path. + */ + internal fun loadFromJson(obj: JSONObject): MhrvConfig { val ids = obj.optJSONArray("script_ids")?.let { arr -> buildList { for (i in 0 until arr.length()) add(arr.optString(i)) } }?.filter { it.isNotBlank() }.orEmpty() @@ -476,6 +531,29 @@ object ConfigStore { "en" -> UiLang.EN else -> UiLang.AUTO }, + frontingGroups = obj.optJSONArray("fronting_groups")?.let { arr -> + buildList { + for (i in 0 until arr.length()) { + val g = arr.optJSONObject(i) ?: continue + val name = g.optString("name").trim() + val ip = g.optString("ip").trim() + val sni = g.optString("sni").trim() + val domArr = g.optJSONArray("domains") + val domains = if (domArr != null) { + buildList { + for (j in 0 until domArr.length()) { + val d = domArr.optString(j).trim() + if (d.isNotEmpty()) add(d) + } + } + } else emptyList() + // Skip half-empty entries — same shape as the + // Rust validator in src/config.rs would reject. + if (name.isEmpty() || ip.isEmpty() || sni.isEmpty() || domains.isEmpty()) continue + add(FrontingGroup(name, ip, sni, domains)) + } + } + }.orEmpty(), ) } } diff --git a/android/app/src/main/java/com/therealaleph/mhrv/CuratedGroups.kt b/android/app/src/main/java/com/therealaleph/mhrv/CuratedGroups.kt new file mode 100644 index 00000000..b5c425a6 --- /dev/null +++ b/android/app/src/main/java/com/therealaleph/mhrv/CuratedGroups.kt @@ -0,0 +1,101 @@ +package com.therealaleph.mhrv + +import android.content.Context +import android.util.Log +import java.io.IOException +import org.json.JSONException +import org.json.JSONObject + +/** + * Loader + merger for the curated fronting-group bundle shipped at + * `assets/fronting-groups/curated.json` (synced from the Rust crate's + * canonical copy at repo-root `assets/fronting-groups/curated.json` by + * the `syncFrontingGroupsAssets` Gradle task). + * + * Same shape as `src/curated_groups.rs` on the Rust side: `mergeInto` + * appends groups whose `name` isn't already present, leaving the user's + * hand-edited entries alone. There's no in-app editor for the entries + * yet, so this is the no-typing path to install Vercel / Fastly / + * AWS-CloudFront / direct-GitHub coverage. + * + * Edge IPs rotate. If a group stops working, the remediation is the + * same as desktop: re-resolve `sni` (`nslookup `) and edit the IP + * by hand in `config.json`. There's no IP-refresh button in the UI yet. + */ +object CuratedGroups { + private const val TAG = "CuratedGroups" + private const val ASSET_PATH = "fronting-groups/curated.json" + + /** Result of [mergeInto], surfaced to the UI for snackbar text. */ + data class MergeReport(val added: Int, val skipped: Int) + + /** + * Read the bundled curated.json from APK assets and parse the + * `fronting_groups` array. Returns null on a packaging or parse + * failure (UI surfaces a generic toast); both failure modes are + * also logged at warn so a user reporting "the button does + * nothing" can be debugged from logcat. Anything else propagates + * — we don't want to swallow `OutOfMemoryError` or a coding bug + * (NPE / IndexOutOfBounds) just because the call site is a + * button-tap. + */ + fun loadCurated(ctx: Context): List? { + val json = try { + ctx.assets.open(ASSET_PATH).bufferedReader().use { it.readText() } + } catch (e: IOException) { + Log.w(TAG, "asset $ASSET_PATH unreadable", e) + return null + } + + val arr = try { + JSONObject(json).optJSONArray("fronting_groups") + } catch (e: JSONException) { + Log.w(TAG, "asset $ASSET_PATH is not valid JSON", e) + return null + } ?: return null + + return buildList { + for (i in 0 until arr.length()) { + val g = arr.optJSONObject(i) ?: continue + val name = g.optString("name").trim() + val ip = g.optString("ip").trim() + val sni = g.optString("sni").trim() + val domArr = g.optJSONArray("domains") ?: continue + val domains = buildList { + for (j in 0 until domArr.length()) { + val d = domArr.optString(j).trim() + if (d.isNotEmpty()) add(d) + } + } + if (name.isEmpty() || ip.isEmpty() || sni.isEmpty() || domains.isEmpty()) continue + add(FrontingGroup(name, ip, sni, domains)) + } + } + } + + /** + * Append every curated group whose `name` isn't already in + * [existing]. Names compare case-insensitively after trim — the + * way humans actually edit configs. Returns a new list (does not + * mutate [existing]) plus a report of how many were added vs. + * already-present. + */ + fun mergeInto( + existing: List, + curated: List, + ): Pair, MergeReport> { + val merged = existing.toMutableList() + var added = 0 + var skipped = 0 + for (g in curated) { + val present = merged.any { it.name.trim().equals(g.name.trim(), ignoreCase = true) } + if (present) { + skipped += 1 + } else { + merged.add(g) + added += 1 + } + } + return merged to MergeReport(added, skipped) + } +} diff --git a/android/app/src/main/java/com/therealaleph/mhrv/ui/HomeScreen.kt b/android/app/src/main/java/com/therealaleph/mhrv/ui/HomeScreen.kt index d228b721..d5e258b8 100644 --- a/android/app/src/main/java/com/therealaleph/mhrv/ui/HomeScreen.kt +++ b/android/app/src/main/java/com/therealaleph/mhrv/ui/HomeScreen.kt @@ -35,6 +35,7 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import com.therealaleph.mhrv.CaInstall import com.therealaleph.mhrv.ConfigStore +import com.therealaleph.mhrv.CuratedGroups import com.therealaleph.mhrv.DEFAULT_SNI_POOL import com.therealaleph.mhrv.MhrvConfig import com.therealaleph.mhrv.Mode @@ -1369,6 +1370,52 @@ private fun AdvancedSettings( Text(stringResource(R.string.adv_upstream_socks5_help)) }, ) + + // Curated fronting-group loader. The bundle ships at + // assets/fronting-groups/curated.json (synced from the Rust + // crate's canonical copy by Gradle's syncFrontingGroupsAssets + // task). Mirrors the desktop UI's Advanced-section button. + // No in-app editor for the entries — this is the no-typing + // path. Existing groups with the same `name` are preserved. + val ctx = LocalContext.current + Column(verticalArrangement = Arrangement.spacedBy(4.dp)) { + Text( + stringResource(R.string.adv_fronting_groups_count, cfg.frontingGroups.size), + style = MaterialTheme.typography.bodyMedium, + ) + Text( + stringResource(R.string.adv_fronting_groups_help), + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + FilledTonalButton( + onClick = { + val curated = CuratedGroups.loadCurated(ctx) + if (curated == null) { + Toast.makeText( + ctx, + ctx.getString(R.string.toast_curated_load_failed), + Toast.LENGTH_LONG, + ).show() + } else { + val (merged, report) = CuratedGroups.mergeInto(cfg.frontingGroups, curated) + onChange(cfg.copy(frontingGroups = merged)) + Toast.makeText( + ctx, + ctx.getString( + R.string.toast_curated_loaded, + report.added, + report.skipped, + ), + Toast.LENGTH_LONG, + ).show() + } + }, + modifier = Modifier.fillMaxWidth(), + ) { + Text(stringResource(R.string.btn_load_curated_groups)) + } + } } } diff --git a/android/app/src/main/res/values-fa/strings.xml b/android/app/src/main/res/values-fa/strings.xml index 9421f805..eb29d4fb 100644 --- a/android/app/src/main/res/values-fa/strings.xml +++ b/android/app/src/main/res/values-fa/strings.xml @@ -75,6 +75,13 @@ upstream_socks5 (اختیاری) اگر تنظیم شود، ترافیک خروجی از این SOCKS5 رد می‌شود. خالی بگذارید برای اتصال مستقیم. + + گروه‌های فرانتینگ: %1$d + بستهٔ آماده شامل Vercel، Fastly (reddit/cnn/python)، AWS CloudFront (netlify) و مسیرهای مستقیم به GitHub است. اگر یک گروه از کار افتاد، آی‌پی را در config.json ویرایش کنید. + بارگذاری گروه‌های فرانتینگ آماده + گروه‌های آماده بارگذاری شد: %1$d مورد افزوده شد، %2$d مورد از قبل وجود داشت. + خواندن فایل گروه‌های فرانتینگ آماده ممکن نشد. + %1$d خط diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 6a7688e7..8e01130b 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -75,6 +75,13 @@ upstream_socks5 (optional) If set, route upstream via this SOCKS5. Leave blank for direct. + + Fronting groups: %1$d + Curated bundle covers vercel, fastly (reddit/cnn/python), AWS CloudFront (netlify), and direct GitHub paths. Edit IPs in config.json if a group stops working. + Load curated fronting groups + Loaded curated groups: %1$d added, %2$d already present. + Could not read curated fronting groups asset. + %1$d lines diff --git a/android/app/src/test/java/com/therealaleph/mhrv/ConfigStoreTest.kt b/android/app/src/test/java/com/therealaleph/mhrv/ConfigStoreTest.kt new file mode 100644 index 00000000..d50d7f1e --- /dev/null +++ b/android/app/src/test/java/com/therealaleph/mhrv/ConfigStoreTest.kt @@ -0,0 +1,113 @@ +package com.therealaleph.mhrv + +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * JVM unit tests for the [ConfigStore.toJson] / [ConfigStore.loadFromJson] + * round trip, with a focus on `fronting_groups` — which the Android UI + * silently dropped on Save before this round of work. These tests exist + * specifically to catch regressions of that data-loss path. + * + * The encode/decode (Base64 + DEFLATE) wrapper around the same JSON is + * not tested here because `android.util.Base64` is stubbed in JVM unit + * tests; the JSON payload it wraps is the same code path covered below. + */ +class ConfigStoreTest { + private val sampleGroups = listOf( + FrontingGroup( + name = "github-direct", + ip = "140.82.121.4", + sni = "github.com", + domains = listOf("gist.github.com"), + ), + FrontingGroup( + name = "vercel", + ip = "76.76.21.21", + sni = "react.dev", + domains = listOf("vercel.com", "vercel.app", "nextjs.org"), + ), + ) + + @Test + fun frontingGroups_roundTripsThroughJson() { + val cfg = MhrvConfig( + mode = Mode.DIRECT, + frontingGroups = sampleGroups, + ) + + val json = cfg.toJson() + val parsed = ConfigStore.loadFromJson(JSONObject(json)) + + assertEquals( + "fronting_groups must round-trip exactly — order, fields, and all", + sampleGroups, + parsed.frontingGroups, + ) + } + + @Test + fun frontingGroups_emptyListProducesNoKey() { + val cfg = MhrvConfig(frontingGroups = emptyList()) + val json = JSONObject(cfg.toJson()) + // Skipping the key when empty matches the pattern used for the + // other optional list fields (passthrough_hosts, sni_hosts) and + // keeps the saved file tidy for users who don't use the feature. + assertTrue( + "fronting_groups should be omitted when the list is empty", + !json.has("fronting_groups"), + ) + } + + @Test + fun frontingGroups_loadIgnoresMalformedEntries() { + // Half-empty entries (missing ip / sni / domains) used to leak + // through if the user hand-edited config.json. The Rust validator + // would reject them at startup; the Kotlin loader skips them on + // read so the UI never sees broken state. + val raw = """ + { + "mode": "direct", + "fronting_groups": [ + {"name": "ok", "ip": "1.2.3.4", "sni": "example.com", + "domains": ["example.com"]}, + {"name": "no-ip", "ip": "", "sni": "x.com", + "domains": ["x.com"]}, + {"name": "no-domains", "ip": "1.2.3.4", "sni": "x.com", + "domains": []}, + {"name": "missing-fields"} + ] + } + """.trimIndent() + + val parsed = ConfigStore.loadFromJson(JSONObject(raw)) + + assertEquals(1, parsed.frontingGroups.size) + assertEquals("ok", parsed.frontingGroups[0].name) + } + + @Test + fun frontingGroups_unknownConfigKeysIgnored() { + // Curated.json carries a `_comment` array that JSONObject would + // happily round-trip if the loader weren't selective. This test + // pins that the loader only reads fields it knows about — same + // defense the Rust serde layer gives us automatically. + val raw = """ + { + "mode": "direct", + "_comment": ["a", "b"], + "fronting_groups": [ + {"name": "g", "ip": "1.2.3.4", "sni": "s.example", + "domains": ["d.example"]} + ] + } + """.trimIndent() + + val parsed = ConfigStore.loadFromJson(JSONObject(raw)) + + assertEquals(1, parsed.frontingGroups.size) + assertEquals(Mode.DIRECT, parsed.mode) + } +} diff --git a/android/app/src/test/java/com/therealaleph/mhrv/CuratedGroupsTest.kt b/android/app/src/test/java/com/therealaleph/mhrv/CuratedGroupsTest.kt new file mode 100644 index 00000000..291f43cd --- /dev/null +++ b/android/app/src/test/java/com/therealaleph/mhrv/CuratedGroupsTest.kt @@ -0,0 +1,78 @@ +package com.therealaleph.mhrv + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Test + +/** + * JVM unit tests for [CuratedGroups.mergeInto]. The asset-loading half + * (`loadCurated`) needs `android.content.Context` and is exercised + * end-to-end by the desktop tests against the canonical curated.json, + * so this file focuses on the merge semantics that decide whether a + * user's hand-edited group survives the curated bundle being applied. + */ +class CuratedGroupsTest { + private val curated = listOf( + FrontingGroup("vercel", "76.76.21.21", "react.dev", listOf("vercel.com")), + FrontingGroup("fastly", "151.101.0.223", "pypi.org", listOf("reddit.com")), + ) + + @Test + fun emptyExisting_addsAllCurated() { + val (merged, report) = CuratedGroups.mergeInto(emptyList(), curated) + + assertEquals(2, report.added) + assertEquals(0, report.skipped) + assertEquals(curated, merged) + } + + @Test + fun nameCollision_preservesUserEntry() { + val userVercel = FrontingGroup( + name = "vercel", + ip = "1.2.3.4", + sni = "user-edited.example", + domains = listOf("user.example"), + ) + val (merged, report) = CuratedGroups.mergeInto(listOf(userVercel), curated) + + assertEquals(1, report.added) + assertEquals(1, report.skipped) + // The user's vercel entry must be untouched — overwriting it + // would silently destroy their hand-tuning. fastly should be + // appended. + val mergedVercel = merged.first { it.name == "vercel" } + assertEquals(userVercel, mergedVercel) + assertNotEquals(curated[0], mergedVercel) + } + + @Test + fun nameMatchIsCaseInsensitive() { + // Real configs end up mixed-case after a copy/paste. "Vercel" / + // " VERCEL " / "vercel" are all the same group as far as the + // matcher is concerned. + val userMixed = FrontingGroup("VERCEL", "1.1.1.1", "x", listOf("x.example")) + val (_, report) = CuratedGroups.mergeInto(listOf(userMixed), curated) + assertEquals(1, report.skipped) + + val userPadded = FrontingGroup(" vercel ", "1.1.1.1", "x", listOf("x.example")) + val (_, paddedReport) = CuratedGroups.mergeInto(listOf(userPadded), curated) + assertEquals( + "Trim should be applied before case-insensitive compare", + 1, paddedReport.skipped, + ) + } + + @Test + fun mergeIsPure_doesNotMutateCallerList() { + val existing = mutableListOf( + FrontingGroup("user-only", "10.0.0.1", "x", listOf("x.example")), + ) + val before = existing.toList() + CuratedGroups.mergeInto(existing, curated) + assertEquals( + "mergeInto must not mutate the caller-supplied existing list", + before, existing, + ) + } +} diff --git a/assets/fronting-groups/curated.json b/assets/fronting-groups/curated.json new file mode 100644 index 00000000..3f19a4db --- /dev/null +++ b/assets/fronting-groups/curated.json @@ -0,0 +1,128 @@ +{ + "_comment": [ + "Curated fronting groups derived from patterniha/MITM-DomainFronting", + "(https://github.com/patterniha/MITM-DomainFronting). The Xray config", + "in that project ships a tested set of (sni, edge, member-domain)", + "tuples for Vercel, Fastly, AWS CloudFront, and direct-to-GitHub paths.", + "This file is the same data, restructured for mhrv-rs's `fronting_groups`", + "config shape. See docs/fronting-groups.md.", + "", + "Edge IPs rotate. If a group stops working, re-resolve `sni`", + "(`nslookup ` or `dig +short `) and replace the IP in-place.", + "Order matters — first group whose `domains` list matches wins, so the", + "more-specific GitHub-direct groups appear before fastly (which would", + "otherwise eat `*.githubusercontent.com` via suffix match).", + "", + "The `_comment` key is ignored by the deserializer (Config doesn't use", + "`deny_unknown_fields`)." + ], + "fronting_groups": [ + { + "name": "github-direct", + "ip": "140.82.121.4", + "sni": "github.com", + "domains": [ + "gist.github.com" + ] + }, + { + "name": "github-content-direct", + "ip": "140.82.121.6", + "sni": "central.github.com", + "domains": [ + "objects-origin.githubusercontent.com" + ] + }, + { + "name": "vercel", + "ip": "76.76.21.21", + "sni": "react.dev", + "domains": [ + "ai-sdk.dev", + "cursor.com", + "err.sh", + "hyper.is", + "nextjs.org", + "now.sh", + "react.dev", + "skills.sh", + "static.fun", + "title.sh", + "turborepo.org", + "vercel-dns.com", + "vercel-status.com", + "vercel.app", + "vercel.blog", + "vercel.com", + "vercel.dev", + "vercel.events", + "vercel.live", + "vercel.pub", + "vercel.sh", + "vercel.store", + "zeit-world.co.uk", + "zeit-world.com", + "zeit-world.net", + "zeit-world.org", + "zeit.co", + "zeit.sh", + "zeitworld.com" + ] + }, + { + "name": "fastly", + "ip": "151.101.0.223", + "sni": "pypi.org", + "domains": [ + "buzzfeed.com", + "cnn.com", + "cnn.io", + "cnn.it", + "cnnarabic.com", + "cnnlabs.com", + "cnnmoney.ch", + "cnnmoney.com", + "cnnmoneystream.com", + "cnnpolitics.com", + "developer.fastly.com", + "fastly-edge.com", + "fastly-terrarium.com", + "fastly.com", + "fastly.io", + "fastly.net", + "fastlylabs.com", + "fastlylb.net", + "github.io", + "githubassets.com", + "githubusercontent.com", + "pinimg.com", + "pinterest.com", + "pypi.org", + "redd.it", + "reddit.app.link", + "reddit.com", + "reddit.map.fastly.net", + "redditblog.com", + "reddithelp.com", + "redditinc.com", + "redditmail.com", + "redditmedia.com", + "redditspace.com", + "redditstatic.com", + "redditstatus.com", + "www.fastly.com", + "www.python.org", + "xtls.github.io" + ] + }, + { + "name": "cloudfront", + "ip": "75.2.60.5", + "sni": "kubernetes.io", + "domains": [ + "netlify.app", + "netlify.com" + ] + } + ] +} diff --git a/config.fronting-groups.example.json b/config.fronting-groups.example.json index a1759dc6..78810530 100644 --- a/config.fronting-groups.example.json +++ b/config.fronting-groups.example.json @@ -1,4 +1,5 @@ { + "_comment": "Domain coverage mirrors assets/fronting-groups/curated.json (the bundle the UI's 'Load curated fronting groups' button installs). Edit IPs in-place if a group stops working — re-resolve `sni` with `nslookup`/`dig` and replace.", "mode": "direct", "google_ip": "216.239.38.120", "front_domain": "www.google.com", @@ -8,57 +9,64 @@ "log_level": "info", "verify_ssl": true, "fronting_groups": [ + { + "name": "github-direct", + "ip": "140.82.121.4", + "sni": "github.com", + "domains": [ + "gist.github.com" + ] + }, + { + "name": "github-content-direct", + "ip": "140.82.121.6", + "sni": "central.github.com", + "domains": [ + "objects-origin.githubusercontent.com" + ] + }, { "name": "vercel", "ip": "76.76.21.21", "sni": "react.dev", "domains": [ - "vercel.com", + "ai-sdk.dev", + "cursor.com", + "err.sh", + "hyper.is", + "nextjs.org", + "now.sh", + "react.dev", + "skills.sh", + "static.fun", + "title.sh", + "turborepo.org", + "vercel-dns.com", + "vercel-status.com", "vercel.app", + "vercel.blog", + "vercel.com", "vercel.dev", + "vercel.events", "vercel.live", + "vercel.pub", "vercel.sh", - "nextjs.org", - "now.sh", - "cursor.com", - "ai-sdk.dev" + "vercel.store", + "zeit-world.co.uk", + "zeit-world.com", + "zeit-world.net", + "zeit-world.org", + "zeit.co", + "zeit.sh", + "zeitworld.com" ] }, { "name": "fastly", - "ip": "151.101.1.140", - "sni": "www.python.org", + "ip": "151.101.0.223", + "sni": "pypi.org", "domains": [ - "redd.it", - "reddit.com", - "redditstatic.com", - "redditmedia.com", - "reddit.app.link", - "redditblog.com", - "reddithelp.com", - "redditinc.com", - "redditmail.com", - "redditspace.com", - "redditstatus.com", - "reddit.map.fastly.net", - - "githubassets.com", - "githubusercontent.com", - "github.io", - - "pypi.org", - - "fastly.com", - "fastly-edge.com", - "fastly-terrarium.com", - "fastly.io", - "fastly.net", - "fastlylabs.com", - "fastlylb.net", - - "www.pinterest.com", - "pinimg.com", - + "buzzfeed.com", "cnn.com", "cnn.io", "cnn.it", @@ -68,14 +76,41 @@ "cnnmoney.com", "cnnmoneystream.com", "cnnpolitics.com", - - "buzzfeed.com" + "developer.fastly.com", + "fastly-edge.com", + "fastly-terrarium.com", + "fastly.com", + "fastly.io", + "fastly.net", + "fastlylabs.com", + "fastlylb.net", + "github.io", + "githubassets.com", + "githubusercontent.com", + "pinimg.com", + "pinterest.com", + "pypi.org", + "redd.it", + "reddit.app.link", + "reddit.com", + "reddit.map.fastly.net", + "redditblog.com", + "reddithelp.com", + "redditinc.com", + "redditmail.com", + "redditmedia.com", + "redditspace.com", + "redditstatic.com", + "redditstatus.com", + "www.fastly.com", + "www.python.org", + "xtls.github.io" ] }, { - "name": "netlify", - "ip": "35.157.26.135", - "sni": "letsencrypt.org", + "name": "cloudfront", + "ip": "75.2.60.5", + "sni": "kubernetes.io", "domains": [ "netlify.app", "netlify.com" diff --git a/docs/fronting-groups.md b/docs/fronting-groups.md index ac57c230..bdee2a2e 100644 --- a/docs/fronting-groups.md +++ b/docs/fronting-groups.md @@ -41,7 +41,43 @@ on that edge through the same tunnel without burning Apps Script quota. `vercel.com` covers both `vercel.com` and `*.vercel.com`. First group in the list whose member matches wins. -A working example is shipped at `config.fronting-groups.example.json`. +A working example is shipped at `config.fronting-groups.example.json`. It +mirrors the same coverage as the curated bundle below. + +## Curated bundle (no-typing path) + +The binary ships [`assets/fronting-groups/curated.json`](../assets/fronting-groups/curated.json) +with five groups derived from the +[`patterniha/MITM-DomainFronting`](https://github.com/patterniha/MITM-DomainFronting) +Xray config — the same set of (sni, edge IP, member-domain) tuples that +project's author has tested in the field: + +| Group | SNI | Covers | +| --- | --- | --- | +| `github-direct` | `github.com` | `gist.github.com` | +| `github-content-direct` | `central.github.com` | `objects-origin.githubusercontent.com` | +| `vercel` | `react.dev` | `vercel.com`, `vercel.app`, `nextjs.org`, `cursor.com`, `zeit.co`, … (29 domains) | +| `fastly` | `pypi.org` | `reddit.com`, `cnn.com`, `pinterest.com`, `buzzfeed.com`, `githubusercontent.com`, `pypi.org`, … (40 domains) | +| `cloudfront` | `kubernetes.io` | `netlify.app`, `netlify.com` | + +The two GitHub-direct groups appear before `fastly` in the list so that +first-match-wins routes `objects-origin.githubusercontent.com` away from +the broader `githubusercontent.com` suffix in `fastly`. + +**Desktop UI** — open the *Advanced* section and click **Load curated +fronting groups**. The button appends groups whose `name` isn't already +in your config; hand-edited entries are never overwritten. Then press +**Save config** to persist. + +**Android UI** — same flow under *Advanced*, **Load curated fronting +groups**. (Android did not round-trip the `fronting_groups` field at all +before this — earlier Android builds silently dropped the field on +Save. If you previously hand-edited groups into `config.json` on a +phone, re-add them or load the curated bundle.) + +**CLI / config-file users** — copy `config.fronting-groups.example.json` +into place, or splice the `fronting_groups` array from +`assets/fronting-groups/curated.json` into your existing `config.json`. ## Picking the (ip, sni) pair diff --git a/src/bin/ui.rs b/src/bin/ui.rs index 7bb46b14..9c30a0fd 100644 --- a/src/bin/ui.rs +++ b/src/bin/ui.rs @@ -1274,6 +1274,51 @@ impl eframe::App for App { Issue #213, #793.", ); }); + + // Curated fronting-group loader. The full list shipped + // in `assets/fronting-groups/curated.json` covers + // Vercel, Fastly (reddit/cnn/python/github-content), + // AWS CloudFront (netlify), and direct-to-GitHub for + // gist + objects-origin. There's no editor for the + // groups in the UI yet — this button is the no-typing + // path to install the full set; hand-edited entries + // are preserved (collision is by group `name`). + ui.horizontal(|ui| { + ui.add_space(120.0 + 8.0); + let count = self.form.fronting_groups.len(); + let label = format!( + "Load curated fronting groups (vercel, fastly, …) · current: {}", + count + ); + if ui.button(label) + .on_hover_text( + "Append the bundled curated fronting groups to your config. \ + Existing groups are preserved — entries with the same `name` \ + are skipped, never overwritten. Press Save config afterwards \ + to persist. Edge IPs may need refresh; see docs/fronting-groups.md." + ) + .clicked() + { + match mhrv_rs::curated_groups::merge_into(&mut self.form.fronting_groups) { + Ok(report) => { + self.toast = Some(( + format!( + "Loaded curated groups: {} added, {} already present. \ + Press Save config to persist.", + report.added, report.skipped, + ), + Instant::now(), + )); + } + Err(e) => { + self.toast = Some(( + format!("Could not load curated groups: {}", e), + Instant::now(), + )); + } + } + } + }); }); }); diff --git a/src/curated_groups.rs b/src/curated_groups.rs new file mode 100644 index 00000000..8d456297 --- /dev/null +++ b/src/curated_groups.rs @@ -0,0 +1,170 @@ +//! Curated fronting groups bundled with the binary. +//! +//! The JSON at `assets/fronting-groups/curated.json` ships a tested set +//! of (sni, edge IP, member-domain) tuples for Vercel, Fastly, AWS +//! CloudFront, and direct-to-GitHub paths — derived from +//! patterniha/MITM-DomainFronting. The UI exposes a button to install +//! these into the user's `fronting_groups` config in one click; CLI +//! users can copy `config.fronting-groups.example.json` (same data). +//! +//! Keep the asset in sync with the example file. `merge_into` is the +//! merge entry point: it appends groups whose `name` isn't already +//! present, leaving the user's hand-edited entries alone. +//! +//! Edge IPs rotate. The `sni` is the source of truth for re-resolution +//! (`nslookup `); see docs/fronting-groups.md. + +use serde::Deserialize; + +use crate::config::FrontingGroup; + +/// Embedded JSON from `assets/fronting-groups/curated.json`. The path +/// is relative to the source file (`src/curated_groups.rs`), so the +/// `..` walks up to the crate root where `assets/` lives. +const CURATED_JSON: &str = include_str!("../assets/fronting-groups/curated.json"); + +#[derive(Debug, Deserialize)] +struct Bundle { + fronting_groups: Vec, +} + +/// Parsed curated fronting groups. Returns the same list every call +/// — cheap enough that we don't bother caching across calls. +pub fn curated_fronting_groups() -> Result, serde_json::Error> { + let bundle: Bundle = serde_json::from_str(CURATED_JSON)?; + Ok(bundle.fronting_groups) +} + +/// Result of a `merge_into` call, surfaced to the UI for toast text. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MergeReport { + /// Groups newly appended to `existing`. + pub added: usize, + /// Groups skipped because a group with the same `name` was already + /// present. The user's entry is left untouched (we never overwrite + /// hand-edits). + pub skipped: usize, +} + +/// Append every curated group whose `name` isn't already in `existing`. +/// Skipped groups are counted in the report. Names compare +/// case-insensitively after trim, matching the way humans edit configs. +pub fn merge_into(existing: &mut Vec) -> Result { + let curated = curated_fronting_groups()?; + let mut report = MergeReport::default(); + for g in curated { + let already = existing + .iter() + .any(|e| e.name.trim().eq_ignore_ascii_case(g.name.trim())); + if already { + report.skipped += 1; + } else { + existing.push(g); + report.added += 1; + } + } + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn curated_bundle_parses() { + let groups = curated_fronting_groups().expect("curated.json must parse"); + assert!( + !groups.is_empty(), + "curated bundle should ship at least one group" + ); + // github-direct must come before fastly, otherwise fastly's + // `githubusercontent.com` suffix would eat + // `objects-origin.githubusercontent.com` before + // github-content-direct gets to claim it. + let pos = |n: &str| groups.iter().position(|g| g.name == n); + let github_content = pos("github-content-direct").expect("github-content-direct present"); + let fastly = pos("fastly").expect("fastly present"); + assert!( + github_content < fastly, + "github-content-direct must precede fastly for first-match-wins" + ); + } + + #[test] + fn merge_into_skips_existing_by_name() { + let mut existing = vec![FrontingGroup { + name: "vercel".into(), + ip: "1.2.3.4".into(), + sni: "user-edited.example".into(), + domains: vec!["user.example".into()], + }]; + let before_len = existing.len(); + let report = merge_into(&mut existing).expect("merge should succeed"); + // The user's vercel entry stays put. + let user_vercel = existing + .iter() + .find(|g| g.name == "vercel") + .expect("user vercel group preserved"); + assert_eq!(user_vercel.ip, "1.2.3.4"); + assert_eq!(user_vercel.sni, "user-edited.example"); + assert_eq!(report.skipped, 1, "vercel collision should be reported"); + assert_eq!(existing.len(), before_len + report.added); + } + + #[test] + fn merge_into_adds_all_when_empty() { + let mut existing: Vec = Vec::new(); + let report = merge_into(&mut existing).expect("merge should succeed"); + assert_eq!(report.skipped, 0); + assert!(report.added > 0); + assert_eq!(existing.len(), report.added); + } + + /// The example config file at the repo root mirrors the curated + /// asset bundle. Both files exist for different audiences (CLI + /// users copy the example, UI users hit the button to load the + /// asset) but their `fronting_groups` payloads must stay identical + /// so the two paths can't drift. This test pins that property. + /// Together with [example_file_loads_through_validate] it also + /// confirms the asset is a valid input to the real load path. + #[test] + fn example_file_mirrors_curated_bundle() { + use crate::config::Config; + let curated = curated_fronting_groups().expect("curated.json parses"); + let example_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("config.fronting-groups.example.json"); + let example_cfg = Config::load(&example_path) + .expect("example file must load + validate"); + assert_eq!( + curated.len(), + example_cfg.fronting_groups.len(), + "curated.json and the example file must declare the same group count" + ); + for (c, e) in curated.iter().zip(example_cfg.fronting_groups.iter()) { + assert_eq!(c.name, e.name, "group name"); + assert_eq!(c.ip, e.ip, "group ip ({})", c.name); + assert_eq!(c.sni, e.sni, "group sni ({})", c.name); + assert_eq!(c.domains, e.domains, "group domains ({})", c.name); + } + } + + /// Run the curated bundle through the same `Config::load` path the + /// CLI and UI use at startup — this exercises the SNI parse, the + /// per-group field validators, and the duplicate-name check inside + /// `validate()`. Catches the failure mode where curated.json and + /// the validator drift apart (e.g. a future validator tightens + /// what counts as a valid SNI but a curated entry slips through + /// because it was only tested against `serde_json::from_str`). + #[test] + fn example_file_loads_through_validate() { + use crate::config::Config; + let example_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("config.fronting-groups.example.json"); + let cfg = Config::load(&example_path) + .expect("example file with curated groups must pass Config::validate"); + assert!( + !cfg.fronting_groups.is_empty(), + "example file should declare fronting groups" + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 6b53a32b..47d072a1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ pub mod cache; pub mod cert_installer; pub mod config; +pub mod curated_groups; pub mod data_dir; pub mod domain_fronter; pub mod lan_utils;