feat(ctypes): add sandboxed link dependencies

doc/changes/added/16248.md

@@ -0,0 +1,2 @@
+- Add Ctypes 0.4 with separate link dependencies and sandboxed executable and
+  C stubs library links (#16248, @rgrinberg)

doc/foreign-code.rst

@@ -137,7 +137,7 @@ file:
 .. code:: dune
 
   (lang dune {{latest}})
-  (using ctypes 0.3)
+  (using ctypes 0.4)
 
 
 Next, here is a ``dune`` file you can use to define an OCaml program that binds
@@ -303,12 +303,17 @@ descriptions by referencing them as the module specified in optional
    - ``(headers (preamble <preamble>)`` adds directly the preamble. Variables
      can be used in ``<preamble>`` such as ``%{read: }``.
 
-- ``(deps <deps-conf list>)`` declares additional dependencies, such as local
-  headers or libraries. Ctypes stub generation is sandboxed, so all such
-  dependencies must be declared. For example, use
+- ``(deps <deps-conf list>)`` declares additional dependencies for Ctypes stub
+  generation and compilation, such as local headers. Ctypes stub generation is
+  sandboxed, so all such dependencies must be declared. For example, use
   ``(deps (source_tree vendor))`` for headers kept in ``vendor``. See
   :doc:`concepts/dependency-spec` for more details.
 
+- ``(link_deps <deps-conf list>)`` declares dependencies for links that consume
+  ``c_library_flags``. This field is available since version 0.4 of the Ctypes
+  extension. These links are sandboxed, so vendored libraries and other local
+  link inputs must be declared.
+
 ``<optional-function-description-fields>`` are:
 
 - ``(concurrency <sequential|unlocked|lwt_jobs|lwt_preemptive>)`` tells ``ctypes

src/dune_rules/ctypes/ctypes_field.ml

@@ -7,7 +7,11 @@ let syntax =
   Dune_lang.Syntax.create
     ~name:(Syntax.Name.parse name)
     ~desc:"the ctypes extension"
-    [ (0, 1), `Deleted_in (3, 11); (0, 2), `Deleted_in (3, 11); (0, 3), `Since (3, 7) ]
+    [ (0, 1), `Deleted_in (3, 11)
+    ; (0, 2), `Deleted_in (3, 11)
+    ; (0, 3), `Since (3, 7)
+    ; (0, 4), `Since (3, 25)
+    ]
 ;;
 
 module Build_flags_resolver = struct
@@ -146,6 +150,7 @@ type t =
   ; generated_types : Module_name.t
   ; generated_entry_point : Module_name.t
   ; deps : Dep_conf.t list
+  ; link_deps : Dep_conf.t list
   ; version : Syntax.Version.t
   }
 
@@ -168,6 +173,10 @@ let decode =
      and+ generated_types = field_o "generated_types" Module_name.decode
      and+ generated_entry_point = field "generated_entry_point" Module_name.decode
      and+ deps = field_o "deps" (repeat Dep_conf.decode)
+     and+ link_deps =
+       field_o
+         "link_deps"
+         (Dune_lang.Syntax.since syntax (0, 4) >>> repeat Dep_conf.decode)
      and+ version = Syntax.get_exn syntax in
      let external_library_name = External_lib_name.of_string external_library_name in
      (match
@@ -196,6 +205,7 @@ let decode =
            ~default:(Module_name.of_checked_string "Types_generated")
      ; generated_entry_point
      ; deps = Option.value ~default:[] deps
+     ; link_deps = Option.value ~default:[] link_deps
      ; version
      })
 ;;

src/dune_rules/ctypes/ctypes_field.mli

@@ -58,6 +58,7 @@ type t =
   ; generated_types : Module_name.t
   ; generated_entry_point : Module_name.t
   ; deps : Dep_conf.t list
+  ; link_deps : Dep_conf.t list
   ; version : Syntax.Version.t
   }
 

src/dune_rules/ctypes/ctypes_rules.ml

@@ -470,6 +470,14 @@ let gen_rules ~cctx ~(buildable : Buildable.t) ~loc ~scope ~dir ~sctx =
      |> Action_builder.write_file target)
 ;;
 
+let link_deps ~expander ~(buildable : Buildable.t) =
+  match buildable.ctypes with
+  | Some ctypes when Dune_lang.Syntax.Version.Infix.(ctypes.version >= (0, 4)) ->
+    Dep_conf_eval.unnamed Sandbox_config.needs_sandboxing ~expander ctypes.link_deps
+  | None | Some _ ->
+    Action_builder.return Env.empty, Sandbox_config.no_special_requirements
+;;
+
 let ctypes_cclib_flags sctx ~expander ~(buildable : Buildable.t) =
   let standard = Action_builder.return [] in
   match buildable.ctypes with

src/dune_rules/ctypes/ctypes_rules.mli

@@ -9,6 +9,11 @@ val gen_rules
   -> sctx:Super_context.t
   -> unit Memo.t
 
+val link_deps
+  :  expander:Expander.t
+  -> buildable:Buildable.t
+  -> Env.t Action_builder.t * Sandbox_config.t
+
 val ctypes_cclib_flags
   :  Super_context.t
   -> expander:Expander.t

src/dune_rules/exe_rules.ml

@@ -248,12 +248,21 @@ let executables_rules
     let* dep_graphs =
       (* Building an archive for foreign stubs, we link the corresponding object
        files directly to improve perf. *)
+      let buildable = exes.buildable in
       let env, sandbox =
         Dep_conf_eval.unnamed
           Sandbox_config.no_special_requirements
           ~expander
           exes.link_deps
       in
+      let ctypes_env, ctypes_sandbox = Ctypes_rules.link_deps ~expander ~buildable in
+      let env =
+        let open Action_builder.O in
+        let+ env = env
+        and+ ctypes_env = ctypes_env in
+        Env.extend_env env ctypes_env
+      in
+      let sandbox = Sandbox_config.inter sandbox ctypes_sandbox in
       let link_args : Command.Args.without_targets Command.Args.t Action_builder.t =
         Command.Args.S
           [ Dyn
@@ -297,7 +306,6 @@ let executables_rules
         |> Action_builder.return
         |> Check_rules.add_files sctx ~dir
       in
-      let buildable = exes.buildable in
       match buildable.ctypes with
       | None ->
         Exe.build_and_link_many

src/dune_rules/lib_rules.ml

@@ -161,6 +161,8 @@ let gen_wrapped_compat_modules (lib : Library.t) cctx =
 let ocamlmklib
       ~loc
       ~c_library_flags
+      ~env
+      ~sandbox
       ~sctx
       ~dir
       ~o_files
@@ -176,8 +178,9 @@ let ocamlmklib
         c_library_flags
         ~f:(cclibs ocaml.lib_config.ccomp_type ~flag:"-ldopt")
     in
-    fun ~custom ~sandbox targets ->
+    fun ~custom ~rule_sandbox targets ->
       let ctx = Super_context.context sctx in
+      let sandbox = Sandbox_config.inter sandbox rule_sandbox in
       [ Command.Args.A "-g"
       ; (if custom then A "-custom" else Command.Args.empty)
       ; A "-o"
@@ -189,7 +192,11 @@ let ocamlmklib
       ; Dyn cclibs
       ; Hidden_targets targets
       ]
-      |> Command.run ~dir:(Path.build (Context.build_dir ctx)) ~sandbox ocaml.ocamlmklib
+      |> Command.run
+           ~dir:(Path.build (Context.build_dir ctx))
+           ~env
+           ~sandbox
+           ocaml.ocamlmklib
       |> Super_context.add_rule sctx ~dir ~loc
   in
   let { Lib_config.ext_lib; ext_dll; _ } = ocaml.lib_config in
@@ -207,11 +214,14 @@ let ocamlmklib
     >>| (function
      | true -> [ static_target; dynamic_target ]
      | false -> [ static_target ])
-    >>= build ~sandbox:Sandbox_config.no_special_requirements ~custom:false
+    >>= build ~rule_sandbox:Sandbox_config.no_special_requirements ~custom:false
   else
     (* Build the static target only by passing the [-custom] flag. *)
     let* () =
-      build ~sandbox:Sandbox_config.no_special_requirements ~custom:true [ static_target ]
+      build
+        ~rule_sandbox:Sandbox_config.no_special_requirements
+        ~custom:true
+        [ static_target ]
     in
     (* The second rule (below) may fail on some platforms, but the build will
        succeed as long as the resulting dynamic library isn't actually needed
@@ -228,7 +238,7 @@ let ocamlmklib
       Context.dynamically_linked_foreign_archives ctx
     in
     Memo.when_ dynamically_linked_foreign_archives (fun () ->
-      build ~sandbox:Sandbox_config.needs_sandboxing ~custom:false [ dynamic_target ])
+      build ~rule_sandbox:Sandbox_config.needs_sandboxing ~custom:false [ dynamic_target ])
 ;;
 
 (* Build a static and a dynamic archive for a foreign library. Note that the
@@ -279,6 +289,8 @@ let foreign_rules (library : Foreign_library.t) ~sctx ~expander ~dir ~dir_conten
     ~archive_name
     ~loc:library.stubs.loc
     ~c_library_flags
+    ~env:(Action_builder.return Env.empty)
+    ~sandbox:Sandbox_config.no_special_requirements
     ~sctx
     ~dir
     ~o_files
@@ -309,6 +321,7 @@ let build_stubs lib ~cctx ~dir ~expander ~requires ~dir_contents ~vlib_stubs_o_f
   then Memo.return ()
   else (
     let modes = Compilation_context.modes cctx |> Option.value_exn in
+    let env, sandbox = Ctypes_rules.link_deps ~expander ~buildable:lib.buildable in
     let ocamlmklib =
       let build_targets_together =
         modes.native
@@ -343,6 +356,8 @@ let build_stubs lib ~cctx ~dir ~expander ~requires ~dir_contents ~vlib_stubs_o_f
         ~sctx
         ~dir
         ~c_library_flags
+        ~env
+        ~sandbox
         ~build_targets_together
     in
     let for_all_modes =

test/blackbox-tests/test-cases/ctypes/delete-0.1.t

@@ -10,6 +10,6 @@ Version 0.1 is deleted.
   2 | (using ctypes 0.1)
                     ^^^
   Error: Version 0.1 of the ctypes extension has been deleted in Dune 3.11.
-  Please port this project to a newer version of the extension, such as 0.3.
+  Please port this project to a newer version of the extension, such as 0.4.
   Hint: You will also need to upgrade to (lang dune 3.7).
   [1]

test/blackbox-tests/test-cases/ctypes/delete-0.2.t

@@ -10,6 +10,6 @@ Version 0.2 is deleted.
   2 | (using ctypes 0.2)
                     ^^^
   Error: Version 0.2 of the ctypes extension has been deleted in Dune 3.11.
-  Please port this project to a newer version of the extension, such as 0.3.
+  Please port this project to a newer version of the extension, such as 0.4.
   Hint: You will also need to upgrade to (lang dune 3.7).
   [1]

test/blackbox-tests/test-cases/ctypes/exe-vendored.t/dune

@@ -4,6 +4,7 @@
  (ctypes
   (external_library_name examplelib)
   (deps (source_tree vendor))
+  ; LINK_DEPS
   (build_flags_resolver
    (vendored
     (c_flags "-Ivendor")

test/blackbox-tests/test-cases/ctypes/exe-vendored.t/dune-project

@@ -1,3 +1,3 @@
-(lang dune 3.7)
+(lang dune 3.25)
 (using ctypes 0.3)
 (use_standard_c_and_cxx_flags false)

test/blackbox-tests/test-cases/ctypes/exe-vendored.t/run.t

@@ -57,3 +57,23 @@ dependencies.
   >    | {kind: "glob", dir: (.dir | basename), predicate}
   >    ]) | unique'
   [{"kind":"glob","dir":"ctypes","predicate":"*.h"}]
+
+Ctypes 0.4 introduces link dependencies and requires the final link to be
+sandboxed.
+
+  $ sed -i.bak 's/(using ctypes 0.3)/(using ctypes 0.4)/' dune-project
+  $ sed -i.bak 's/; LINK_DEPS/(link_deps (source_tree vendor))/' dune
+  $ rm dune-project.bak dune.bak
+  $ DYLD_LIBRARY_PATH="$TARGET" LD_LIBRARY_PATH="$TARGET" dune exec ./example.exe
+  4
+  $ dune trace cat | jq_dune -sc '
+  >   [ .[]
+  >   | processes
+  >   | select((.args.target_files // [])
+  >            | index("_build/default/example.exe"))
+  >   | { sandbox: (.args.dir | contains(".sandbox"))
+  >     , example_flags:
+  >         ([.args.process_args[] | select(. == "-lexample")] | length)
+  >     }
+  >   ][0]'
+  {"sandbox":true,"example_flags":1}

test/blackbox-tests/test-cases/ctypes/lib-vendored.t/dune-project

@@ -1,3 +1,3 @@
-(lang dune 3.7)
+(lang dune 3.25)
 (using ctypes 0.3)
 (use_standard_c_and_cxx_flags false)

test/blackbox-tests/test-cases/ctypes/lib-vendored.t/run.t

@@ -14,19 +14,35 @@ This is the version that builds into a library.
 With Ctypes 0.3, the C stubs library link runs outside a sandbox.
 
   $ dune trace cat | jq_dune -sc '
-  >   [.[] | processes] as $processes
-  >   | { sandbox:
-  >         ([$processes[]
-  >          | select((.args.target_files // [])
-  >                   | any(contains("dllexamplelib_stubs")))
-  >          | (.args.dir | contains(".sandbox"))
-  >          ][0])
+  >   [ .[]
+  >   | processes
+  >   | select((.args.target_files // [])
+  >            | any(contains("dllexamplelib_stubs")))
+  >   | { sandbox: (.args.dir | contains(".sandbox"))
   >     , example_flags:
-  >         ([$processes[]
-  >          | select((.args.target_files // [])
-  >                   | index("_build/default/stubgen/examplelib.cmxa"))
-  >          | [.args.process_args[] | select(. == "-lexample")] | length
-  >          ][0])
-  >     }'
+  >         ([.args.process_args[] | select(. == "-lexample")] | length)
+  >     }
+  >   ][0]'
   {"sandbox":false,"example_flags":1}
 
+Ctypes 0.4 introduces link dependencies and requires the C stubs library link
+to be sandboxed.
+
+  $ sed -i.bak 's/(using ctypes 0.3)/(using ctypes 0.4)/' dune-project
+  $ sed -i.bak \
+  >   's/; LINK_DEPS/(link_deps (source_tree vendor))/' stubgen/dune
+  $ rm dune-project.bak stubgen/dune.bak
+  $ DYLD_LIBRARY_PATH="$TARGET" LD_LIBRARY_PATH="$TARGET" dune exec ./example.exe
+  4
+  $ dune trace cat | jq_dune -sc '
+  >   [ .[]
+  >   | processes
+  >   | select((.args.target_files // [])
+  >            | any(contains("dllexamplelib_stubs")))
+  >   | { sandbox: (.args.dir | contains(".sandbox"))
+  >     , example_flags:
+  >         ([.args.process_args[] | select(. == "-lexample")] | length)
+  >     }
+  >   ][0]'
+  {"sandbox":true,"example_flags":1}
+

test/blackbox-tests/test-cases/ctypes/lib-vendored.t/stubgen/dune

@@ -4,6 +4,7 @@
  (ctypes
   (external_library_name examplelib)
   (deps (source_tree vendor))
+  ; LINK_DEPS
   (build_flags_resolver
    (vendored
     ;; hack: multiple -I directives to work around cc commands being run from