Mob.Certs + jniLibs-via-mob_beam for runtime escript/rebar3 on Android

android/jni/mob_beam.zig

@@ -296,6 +296,16 @@ export fn mob_start_beam(app_module: [*:0]const u8) callconv(.c) void {
     _ = jni.setenv("ERL_CRASH_DUMP", crash_dump, 1);
     _ = jni.setenv("ERL_CRASH_DUMP_SECONDS", "30", 1);
 
+    // MOB_NATIVE_LIB_DIR — the app's nativeLibraryDir (apk_data_file context,
+    // exec allowed). Apps that bundle extra binaries (escript, rebar3, etc.)
+    // as `lib<name>.so` in jniLibs/<abi>/ can find them here at runtime —
+    // their paths include the APK install hash and aren't predictable at
+    // compile time. Empty when launched from a split APK that didn't extract
+    // .so files; callers should fall back to BINDIR in that case.
+    if (s_native_lib_dir[0] != 0) {
+        _ = jni.setenv("MOB_NATIVE_LIB_DIR", jni.asCStr(&s_native_lib_dir), 1);
+    }
+
     // RUSTLER_BEAM_LIBRARY_PATH — tells rustler where the .so containing it
     // (libpigeon.so in Mob's static-link model) lives, so its
     // DlsymNifFiller can dlopen(path, RTLD_NOW | RTLD_NOLOAD) directly
@@ -543,6 +553,37 @@ export fn mob_start_beam(app_module: [*:0]const u8) callconv(.c) void {
         }
     }
 
+    // Optional ERTS extras: symlink iff the app shipped them in jniLibs.
+    // Silently skip otherwise — these aren't required for BEAM boot, but apps
+    // that want them (e.g. Mix.install of a rebar3-built dep needs `escript`
+    // *and* a spawnable `erl` / `erlexec` for the escript runner to bootstrap
+    // a fresh VM) can drop `lib<name>.so` into android/app/src/main/jniLibs/<abi>/
+    // to get a working BINDIR/<name>. `erl` and `erlexec` both target the
+    // same library because they're the same binary — erlexec doesn't switch
+    // on argv[0].
+    if (s_native_lib_dir[0] != 0) {
+        const opt_exes = [_][*:0]const u8{ "escript", "erlexec", "erl", "beam.smp" };
+        const opt_libs = [_][*:0]const u8{ "libescript.so", "liberlexec.so", "liberlexec.so", "libbeam_smp.so" };
+        var j: usize = 0;
+        while (j < opt_exes.len) : (j += 1) {
+            var bin_path_buf: [512]u8 = undefined;
+            var lib_path_buf: [512]u8 = undefined;
+            const bin_path = formatZ(&bin_path_buf, "{s}/{s}/bin/{s}", .{ otp_root, ERTS_VSN, opt_exes[j] });
+            const lib_path = formatZ(&lib_path_buf, "{s}/{s}", .{ jni.asCStr(&s_native_lib_dir), opt_libs[j] });
+            var st: jni.Stat = undefined;
+            if (jni.stat(lib_path, &st) == 0) {
+                _ = jni.unlink(bin_path);
+                if (jni.symlink(lib_path, bin_path) == 0) {
+                    logi("mob_start_beam: symlink {s} -> {s} (optional)", .{ opt_exes[j], lib_path });
+                } else {
+                    loge("mob_start_beam: symlink {s} failed: {s}", .{ opt_exes[j], lastErrno() });
+                }
+            }
+            // No lib in nativeLibDir => app didn't ask for this extra. Skip
+            // silently — don't log; not an error.
+        }
+    }
+
     // Symlink sqlite3_nif.so into the exqlite OTP lib structure so that
     // code:priv_dir(:exqlite) resolves correctly.
     //

common_fixes.md

@@ -1042,3 +1042,112 @@ once the app is foregrounded or running under a foreground
 service. Symptom: any socket attempt returns `:closed` / `:timeout`
 immediately. Bring the app foreground or attach a foreground
 service before triggering long-lived network work.
+
+## `FunctionClauseError` in `pubkey_os_cacerts.conv_error_reason/1` (Android, missing CA bundle)
+
+**Symptom** — In a mob app on a real Android device, an HTTPS request via
+Req / Mint / Finch / anything using OTP-26+ default `:ssl` opts crashes:
+
+```
+** (FunctionClauseError) no function clause matching in
+   :pubkey_os_cacerts.conv_error_reason/1
+   (public_key 1.21) :pubkey_os_cacerts.conv_error_reason(:no_cacerts_found)
+```
+
+Hex itself doesn't hit this — it bakes its own CA bundle into
+`Hex.HTTP.SSL` — so `mix install/2` may succeed where the *first* call
+from a user dep fails. The crash also doesn't appear on iOS or on the
+Android emulator (their `:ssl` defaults pick up the OS trust store).
+
+**Root cause** — `:public_key.cacerts_load/0` (called by `:ssl`'s
+defaults at OTP 26+) probes `/etc/ssl/certs/ca-certificates.crt` and a
+handful of distro-specific paths. None of those exist on Android — the
+system trust store lives behind a Java API that BEAM's `:public_key`
+doesn't reach. `cacerts_get/0` then raises with `no_cacerts_found`, and
+in some OTP versions `pubkey_os_cacerts.conv_error_reason/1` doesn't
+have a clause for that — the surface error becomes a
+`FunctionClauseError` rather than the cleaner `no_cacerts_found`.
+
+**Fix** — Bundle a CA-bundle PEM in your app `priv/` (the conventional
+source is the `castore` hex package — copy `cacerts.pem` into your priv
+at build time) and call `Mob.Certs.load_cacerts!/1` once at startup
+*before* anything tries TLS:
+
+```elixir
+def on_start do
+  Mob.Certs.load_cacerts!(Application.app_dir(:my_app, "priv/cacerts.pem"))
+  # …rest of startup…
+end
+```
+
+See the `Mob.Certs` moduledoc for the rationale, the cross-platform
+notes (iOS and the Android emulator don't need this, but calling
+unconditionally is safe), and the available functions.
+
+## Runtime `Mix.install` of a rebar3-built dep fails on Android (telemetry / jose / jiffy / …)
+
+**Symptom** — In a notebook setup cell on a real mob app (Livebook-style),
+`Mix.install/2` of anything that transitively pulls a rebar3-built
+Erlang dep — `telemetry` is the common one (Req → Mint → telemetry,
+Phoenix → telemetry, etc.) — crashes:
+
+```
+** (Mix.Error) Could not compile dependency :telemetry,
+"/data/user/0/<pkg>/files/livebook/mix_home/elixir/1-19-otp-29/rebar3 bare compile --paths …"
+command failed.
+   (mix 1.19.5) lib/mix/tasks/deps.compile.ex:276: Mix.Tasks.Deps.Compile.do_rebar3/2
+```
+
+Pure-Mix deps install fine. The rebar3 path fails because there is no
+`rebar3` binary at the expected `$MIX_HOME` location, and even if you
+copy one there, `app_data_file` SELinux context blocks `execve()` of
+files under the app's writable storage. Same wall as `inet_gethost`
+hits — and the fix is the same JNI-extracted-shared-lib trick.
+
+**Fix** — Bundle the chain ERTS needs to spawn a fresh BEAM, plus
+rebar3's escript, as `lib<name>.so` files in your app's
+`android/app/src/main/jniLibs/<abi>/`:
+
+| File in jniLibs | What it is | Source |
+|---|---|---|
+| `libescript.so`     | OTP escript runner | `$OTP_ANDROID/erts-<vsn>/bin/escript` |
+| `liberlexec.so`     | BEAM launcher (also serves as `erl`) | `$OTP_ANDROID/erts-<vsn>/bin/erlexec` |
+| `libbeam_smp.so`    | The BEAM VM itself (~32 MB unstripped) | `$OTP_ANDROID/erts-<vsn>/bin/beam.smp` |
+| `librebar3_data.so` | rebar3's escript archive | `~/.mix/elixir/<vsn>/rebar3` |
+| `librebar3.so`      | tiny `/system/bin/sh` wrapper (see below) | written at build time |
+
+`mob_beam.zig`'s optional-symlink list picks these up at boot:
+`BINDIR/escript`, `BINDIR/erl`, `BINDIR/erlexec`, and `BINDIR/beam.smp`
+all become `apk_data_file`-context-exec-able. `MOB_NATIVE_LIB_DIR`
+exposes the nativeLibDir path so the app can reach the bundled files
+at runtime.
+
+The rebar3 wrapper (`librebar3.so` content):
+
+```sh
+#!/system/bin/sh
+exec "${BINDIR}/escript" "${MOB_DATA_DIR}/rebar3" "$@"
+```
+
+…plus an app-side step at startup to symlink `librebar3_data.so` to a
+filename of `rebar3` (escript derives the module name from the
+file's basename, and rebar3's archive exports `rebar3:main/1`):
+
+```elixir
+File.cp_r!(Path.join([System.get_env("MOB_NATIVE_LIB_DIR"), "librebar3_data.so"]),
+           Path.join([System.get_env("MOB_DATA_DIR"), "rebar3"]))
+System.put_env("MIX_REBAR3", Path.join([System.get_env("MOB_NATIVE_LIB_DIR"), "librebar3.so"]))
+```
+
+`$OTP_ROOT/bin/<name>.boot` symlinks (`no_dot_erlang.boot`, `start.boot`)
+are normally created by standard OTP's `bin/Install` script — mob's
+deploy skips that. Materialize them lazily at app boot.
+
+Verified end-to-end on a Moto G Power 5G 2024 (Android 14):
+`Mix.install([{:req, "~> 0.5"}])` resolves Req's full tree, compiles
+`telemetry` via on-device rebar3, and a follow-up `Req.get!/2` returns
+real JSON over real TLS.
+
+**Trade-off** — ~33 MB APK size for the bundled `beam.smp`. Pure-Mix
+deps don't need any of this; if your app never runs rebar3 deps at
+runtime, skip the whole bundling and avoid the cost.

lib/mob/certs.ex

@@ -0,0 +1,110 @@
+defmodule Mob.Certs do
+  @moduledoc """
+  CA-certificate loading for mob apps. Companion to `Mob.DNS` — same
+  shape: a small wrapper documenting and working around something OTP
+  assumes about the OS that Android doesn't satisfy.
+
+  ## Why this exists
+
+  `:public_key.cacerts_load/0` looks for a system CA bundle at one of
+  the distro paths it knows (`/etc/ssl/certs/ca-certificates.crt`,
+  `/etc/pki/tls/certs/ca-bundle.crt`, `/etc/ssl/cert.pem`, …). On
+  Android none of those exist — the system trust store lives behind a
+  Java API that BEAM's `:public_key` doesn't reach. Subsequent calls
+  to `:public_key.cacerts_get/0` therefore raise with `no_cacerts_found`,
+  and any library that consults it (Req → Mint → `:ssl`, Finch, anything
+  using OTP-26+ default `:ssl` opts) crashes on the first TLS connect.
+
+  Adding insult: in some OTP versions `pubkey_os_cacerts.conv_error_reason/1`
+  has no clause for `no_cacerts_found`, so the surface error is a
+  `FunctionClauseError` — opaque to the unsuspecting reader. The fix is
+  the same regardless: load a PEM bundle into `:public_key` once at boot.
+
+  Hex itself bakes its own DER bundle, so the BEAM can `mix.install/2`
+  without this fix; every other Elixir HTTP library can't.
+
+  ## What to do
+
+  Bundle a CA PEM in your app priv (e.g. copy `castore`'s `cacerts.pem`)
+  and call `Mob.Certs.load_cacerts!/1` once at boot, *before* anything
+  tries TLS:
+
+      def on_start do
+        Mob.Certs.load_cacerts!(Application.app_dir(:my_app, "priv/cacerts.pem"))
+        # …rest of startup…
+      end
+
+  The bundle is the app's choice — security: who do you trust. The
+  conventional source is the `castore` hex package (a current Mozilla
+  trust store), copied into `priv/` at build time.
+
+  iOS isn't affected — Darwin exposes the trust store via the paths
+  Erlang knows about, so `:public_key.cacerts_load/0` (no arg) works
+  there. Calling `load_cacerts!/1` on iOS at the bundled-PEM path is a
+  harmless extra load; cross-platform apps can call it unconditionally.
+
+  ## Scope
+
+  - Loads CA certificates from a PEM file path.
+  - Wraps `:public_key.cacerts_load/1` so failure shapes are predictable
+    (`{:error, reason}` rather than the OTP-version-dependent
+    `FunctionClauseError` you sometimes see otherwise).
+  - Pure Elixir. No NIF, no platform branch.
+  """
+
+  @doc """
+  Load CA certs from a PEM file into Erlang's `:public_key` cacert store.
+
+  Idempotent: re-loading the same bundle just re-merges its certs into
+  the in-process trust store; no duplication, no error.
+
+  Returns `:ok` on success or `{:error, reason}` if the file can't be
+  read or parsed.
+
+      iex> Mob.Certs.load_cacerts("priv/cacerts.pem")
+      :ok
+
+  """
+  @spec load_cacerts(Path.t()) :: :ok | {:error, term()}
+  def load_cacerts(path) when is_binary(path) do
+    case :public_key.cacerts_load(String.to_charlist(path)) do
+      :ok -> :ok
+      {:error, _} = err -> err
+    end
+  end
+
+  @doc """
+  Same as `load_cacerts/1`, but raises on failure.
+
+  Use this at boot when failing-to-load is unrecoverable — i.e. when the
+  app needs HTTPS at all to function. Most callers want this variant.
+  """
+  @spec load_cacerts!(Path.t()) :: :ok
+  def load_cacerts!(path) when is_binary(path) do
+    case load_cacerts(path) do
+      :ok ->
+        :ok
+
+      {:error, reason} ->
+        raise "Mob.Certs.load_cacerts!/1 failed for #{inspect(path)}: " <>
+                inspect(reason)
+    end
+  end
+
+  @doc """
+  True if any CA certificates are loaded in the `:public_key` store.
+
+  Useful for diagnostics and tests. `:public_key.cacerts_get/0` raises
+  when nothing is loaded; `loaded?/0` catches that and returns `false`
+  instead.
+  """
+  @spec loaded?() :: boolean()
+  def loaded? do
+    case :public_key.cacerts_get() do
+      [_ | _] -> true
+      [] -> false
+    end
+  rescue
+    _ -> false
+  end
+end

mix.exs

@@ -54,7 +54,10 @@ defmodule Mob.MixProject do
 
   def application do
     [
-      extra_applications: [:logger]
+      # :public_key is needed by Mob.Certs at runtime; Elixir 1.19+ strips
+      # unused OTP applications from the code path, so it must be declared
+      # here even though mob doesn't *start* it directly.
+      extra_applications: [:logger, :public_key]
     ]
   end