diff --git a/.gitignore b/.gitignore index f8b7bc13a..e63cd9a25 100644 --- a/.gitignore +++ b/.gitignore @@ -140,6 +140,9 @@ zig/transpile-test.zig zig/zig-out zig/.zig-cache zig/.zig-cache-* +# Per-build Zig/CLEAR caches. 854 of these had been committed by accident. +zig/.zig-global-cache +zig/.clear-module-cache zig/.clear-cache zig/.clear-transpile-cache zig/fiber-stack-check/pass/build @@ -219,3 +222,4 @@ compiler/.ruby-rbs/ compiler/.ruby-original/ # Local Spinel fork used for the AOT experiment tmp/spinel/ + diff --git a/CLAUDE.md b/CLAUDE.md index 4eed4c8e4..a59ab463e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -113,6 +113,7 @@ Reference docs: `mir-bugs.md` (known MIR violations), `alloc-bugs.md` (frame-the **Sigils:** `$` pipeline/interp, `&` mutation, `|>` SMOOTH (safe pipeline w/ error prop), `_` placeholder. **Tense Sigils:** `!` = Error / Error handling, `?` = Option / nil handling, `~` = Stream / future handling. +**Sigils:** `$` pipeline/interp, `&` explicit mutable call-site path, `|>` SMOOTH (safe pipeline w/ error prop), `_` placeholder, `TRY` explicit propagation. **Ownership / capabilities — bindings, not types.** Two sigil groups: - **Group 1 (sync / ownership wrappers):** `@locked`, `@writeLocked`, `@shared` (Arc), `@multiowned` (Rc), `@local`. Stored on `SymbolEntry#sync` and `#storage`. Composed via `MIR::CapWrap`. diff --git a/clear b/clear index 061f6eb1e..c447146a3 100755 --- a/clear +++ b/clear @@ -34,7 +34,9 @@ require 'set' require 'json' require 'rbconfig' require 'open3' +require 'etc' require_relative 'compiler/ruby/tools/clear_build_support' +require_relative 'compiler/ruby/compiler/package_source' require_relative 'tools/zig_coverage_support' # Coverage bootstrap MUST run before any compiler/ruby require so SimpleCov can @@ -251,7 +253,7 @@ end # ------------------------------------------------------------------------- # Build: transpile + compile # ------------------------------------------------------------------------- -def incremental_transpile_runner(source:, source_dir:, pkg_paths:, use_c_allocator:, use_debug_allocator:, default_stack:, ownership_mode:, transpile_flag:, cache_path:) +def incremental_transpile_runner(source:, source_dir:, pkg_paths:, use_c_allocator:, use_debug_allocator:, default_stack:, main_tier:, ownership_mode:, transpile_flag:, cache_path:) require_relative 'compiler/ruby/incremental' fingerprint = Digest::SHA256.hexdigest([ ClearBuildSupport.compiler_signature(BUILD_SUPPORT_CONFIG), @@ -269,6 +271,7 @@ def incremental_transpile_runner(source:, source_dir:, pkg_paths:, use_c_allocat use_c_allocator: use_c_allocator, use_debug_allocator: use_debug_allocator, default_stack: default_stack, + main_tier: main_tier, ownership_mode: ownership_mode ), module_path: source, @@ -373,7 +376,16 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo package_imports = closure_paths.flat_map do |dep_path| File.read(dep_path).scan(/REQUIRE\s+"pkg:([^"]+)"(?:\s+AS\s+([A-Za-z_]\w*))?/) end.uniq - pkg_requires = pkg_paths.keys + # A member of a multi-file package compiles as part of that unit, never on + # its own -- transpiling it standalone as well declares everything in it + # twice, and its sibling REQUIREs cannot resolve without the group anyway. + grouped_member_paths = pkg_paths.values + .select { |spec| spec.to_s.include?(",") } + .flat_map { |spec| spec.to_s.split(",").map { |m| File.expand_path(m.strip) } } + .to_set + pkg_requires = pkg_paths.reject { |_name, spec| + !spec.to_s.include?(",") && grouped_member_paths.include?(File.expand_path(spec.to_s)) + }.keys pkg_flags = pkg_paths.map do |pkg_name, pkg_path| "--pkg #{pkg_name}=#{pkg_path}" end.join(" ") @@ -385,6 +397,7 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo build_dir = coverage_module_mode ? ZIG_DIR : File.join(ZIG_DIR, ".clear-cache", cache_key) cleanup_paths = [] FileUtils.mkdir_p(build_dir) + ClearBuildSupport.prune_build_cache!(File.join(ZIG_DIR, ".clear-cache"), keep: build_dir) unless coverage_module_mode unless coverage_module_mode ClearBuildSupport.ensure_symlink(File.join(build_dir, 'runtime'), File.join(ZIG_DIR, 'runtime')) ClearBuildSupport.ensure_symlink(File.join(build_dir, 'lib'), File.join(ZIG_DIR, 'lib')) @@ -393,6 +406,17 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo cache_dir = coverage_module_mode ? File.join(ZIG_DIR, ".clear-cache", "#{cache_key}-coverage-cache") : File.join(build_dir, '.zig-cache') global_cache_dir = coverage_module_mode ? File.join(ZIG_DIR, ".clear-cache", "#{cache_key}-coverage-global-cache") : File.join(build_dir, '.global-zig-cache') + # Per-REQUIRE-unit cache. The transpile cache above keys the whole program + # on all of its sources, so one edit recompiles every imported module; this + # one keeps the modules that edit did not reach. Shared across roots, and + # read by ModuleImporter in-process or in the transpiler subprocess. + unless bypass_transpile_cache + ENV['CLEAR_MODULE_CACHE_DIR'] ||= File.join(ZIG_DIR, '.clear-module-cache') + ENV['CLEAR_MODULE_CACHE_KEY'] ||= Digest::SHA256.hexdigest( + [ClearBuildSupport.compiler_signature(BUILD_SUPPORT_CONFIG), transpile_flag].join("\0") + ) + end + tmp_name = coverage_module_mode ? "._clear_cov_#{base_name}_#{$$}.zig" : "._clear_tmp_#{base_name}.zig" tmp_zig = File.join(build_dir, tmp_name) @@ -404,6 +428,7 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo use_c_allocator: use_c_allocator, use_debug_allocator: use_debug_allocator, default_stack: default_stack, + main_tier: main_tier, ownership_mode: ownership_mode, transpile_flag: transpile_flag, cache_path: File.join(build_dir, 'root.clearc') @@ -447,6 +472,12 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo zig_code = zig_code.gsub("@import(\"#{zig_name}.zig\")", "@import(\"#{pkg_name}.zig\")") end end + # A member of a multi-file package is emitted as its OWNER's module (only the + # owner is built), and the owner may never appear in a REQUIRE, so the scan + # above does not know its name. Map every built package name too. + pkg_requires.each do |pkg_name| + zig_code = zig_code.gsub("@import(\"#{pkg_name}\")", "@import(\"#{pkg_name}.zig\")") + end # EXTERN ... FROM "cheat_runtime" emits `@import("cheat_runtime")`, but in # the standalone (`./clear build`) flow there is no Zig module by that # name -- the runtime is included as a relative file. Map the import to @@ -490,22 +521,42 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo end # Transpile package modules to .zig files - pkg_modules = pkg_requires.map do |pkg_name| + build_pkg_module = lambda do |pkg_name| pkg_path = ClearBuildSupport.find_package_source(pkg_name, start_dir: source_dir) error "Package '#{pkg_name}' not found from #{source_dir}" unless pkg_path # Collect transitive package deps for nested REQUIRE "pkg:..." - pkg_src = File.read(pkg_path) - nested_imports = pkg_src.scan(/REQUIRE\s+"pkg:([^"]+)"(?:\s+AS\s+([A-Za-z_]\w*))?/) - nested_pkgs = nested_imports.map(&:first).uniq - nested_flags = nested_pkgs.map { |np| - np_path = ClearBuildSupport.find_package_source(np, start_dir: File.dirname(pkg_path)) - error "Package '#{np}' not found from #{File.dirname(pkg_path)}" unless np_path - "--pkg #{np}=#{np_path}" - }.join(" ") + # A multi-file package registers its members as one comma-joined spec, so + # scan every member -- reading the spec as a single path raises ENOENT. + pkg_members = pkg_path.split(',').map(&:strip) + # A multi-file package is ONE compilation unit. Transpiling a single member + # makes its sibling REQUIREs resolve back to the whole package, which then + # declares that member twice; merging first is what the importer does for + # the same reason. + pkg_root = pkg_members.first + pkg_src = if pkg_members.length > 1 + merged = PackageSource.merge(pkg_members, resolve_pkg: ->(name) { pkg_paths[name] }) + pkg_root = File.join(build_dir, "#{pkg_name}.merged.clear") + ClearBuildSupport.write_if_changed(pkg_root, merged.source) + merged.source + else + File.read(pkg_root) + end + # A package's own REQUIREs reach further packages, so pass the whole + # transitive closure: one level leaves the sub-transpile unable to resolve + # anything a nested package itself requires. Keep this package's own entry + # too -- for a multi-file unit that registration is what tells the importer + # its members belong together. + # Import-rewrite below needs the (pkg, alias) pairs, not just the names. + nested_imports = pkg_src.scan(/REQUIRE\s+"pkg:([^"]+)"(?:\s+AS\s+([A-Za-z_]\w*))?/).uniq + nested_flags = pkg_members + .flat_map { |member| ClearBuildSupport.collect_package_dependencies(member).to_a } + .uniq { |np, _| np } + .map { |np, np_path| "--pkg #{np}=#{np_path}" } + .join(" ") begin pkg_zig, _pkg_cache_file = ClearBuildSupport.transpile_cached( config: BUILD_SUPPORT_CONFIG, - source_path: pkg_path, + source_path: pkg_root, mode: :module, transpile_flag: "--module #{nested_flags}".strip, source_text: pkg_src, @@ -523,6 +574,11 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo pkg_zig = pkg_zig.gsub("@import(\"#{zig_name}\")", "@import(\"#{np}.zig\")") pkg_zig = pkg_zig.gsub("@import(\"#{zig_name}.zig\")", "@import(\"#{np}.zig\")") end + # A member of a multi-file package is emitted as its OWNER's module (only + # the owner is built), and the owner may never appear in a REQUIRE here. + pkg_requires.each do |built| + pkg_zig = pkg_zig.gsub("@import(\"#{built}\")", "@import(\"#{built}.zig\")") + end # A package can own EXTERN ... FROM "module" declarations; its emitted # named imports must resolve to the FFI files copied into the build dir. ffi_modules.each do |m, _src_mod| @@ -538,6 +594,18 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo pkg_name end + # Warm the content-addressed transpile cache first. Each package is an + # independent transpile, so forked workers populate exactly what the serial + # pass below then reads back as cache hits -- the same shape as incremental + # compilation, and parallelism stays at this one call site. + ClearBuildSupport.prewarm_in_parallel( + pkg_requires, + jobs: (ENV['CLEAR_JOBS'] || Etc.nprocessors).to_i.clamp(1, 32), + &build_pkg_module + ) + + pkg_modules = pkg_requires.map(&build_pkg_module) + # Publish the root last. A persistent Zig watcher may react to every rename; # writing dependencies first guarantees that it never observes a new root # paired with stale generated package or FFI modules. @@ -598,7 +666,13 @@ def do_build(source, output: nil, opt_level: 'Debug', extra_flags: [], module_mo # experimental incremental path. Only the persistent watch command opts in. cmd_parts << '-fno-incremental' - unless module_mode + if module_mode + # `clear test` builds Debug by default. An explicit level lets a suite run + # the LLVM backend instead of the self-hosted one -- which is where the + # lexer keyword miscompile lived -- and turns on the safety checks that a + # Debug arena hides. + cmd_parts += ['-O', opt_level] unless opt_level == 'Debug' + else bin_name = "#{File.basename(output)}-#{$$}" cmd_parts += ['-O', opt_level] + extra_flags cmd_parts += ['-fno-strip'] if profile @@ -896,6 +970,7 @@ when 'build', 'watch' # Parse build flags remaining = [] stack_check = nil # auto: on for release/safe, off for debug + main_tier_override = nil # --main-tier: the recursive self-hosted parser needs more than the 64KB debug default i = 0 while i < args.length case args[i] @@ -921,6 +996,11 @@ when 'build', 'watch' when '--no-stack-check' stack_check = false # explicit override i += 1 + when '--main-tier' + tier_arg = args[i + 1] + error "--main-tier needs a tier (micro|standard|large|xl|service)" unless tier_arg + main_tier_override = tier_arg.downcase.to_sym + i += 2 when '--force' @force_build = true i += 1 @@ -1028,7 +1108,7 @@ when 'build', 'watch' exit 0 end - result = do_build(source, output: output, opt_level: opt_level, extra_flags: extra_flags, default_stack: default_stack, force: !!@force_build, use_c_allocator: use_c_allocator, use_debug_allocator: use_debug_allocator, bypass_transpile_cache: bypass_transpile_cache, ownership_mode: ownership_mode) + result = do_build(source, output: output, opt_level: opt_level, extra_flags: extra_flags, default_stack: default_stack, force: !!@force_build, use_c_allocator: use_c_allocator, use_debug_allocator: use_debug_allocator, bypass_transpile_cache: bypass_transpile_cache, ownership_mode: ownership_mode, main_tier: main_tier_override) exit 0 if result == :up_to_date puts "Built: #{output_path}" @@ -1291,6 +1371,15 @@ when 'test' exec(RbConfig.ruby, compat_script, *compat_args) end + # Opt into an optimized test build. ReleaseSafe keeps the safety checks and + # routes through LLVM rather than the self-hosted backend. + test_opt_level = if test_args.delete('--safe') + 'ReleaseSafe' + elsif test_args.delete('--optimized') + 'ReleaseFast' + else + 'Debug' + end profile_mode = test_args.delete('--profile') strict_mode = test_args.delete('--strict') frame_debug = test_args.delete('--debug-frame') || test_args.delete('--no-frame') @@ -1472,17 +1561,14 @@ when 'test' end zig_code = zig_code.gsub('@import("runtime-header.zig")', '@import("runtime/runtime-header.zig")') - # Write to a build dir matching normal build layout. Transpiled tests - # import runtime/runtime-header.zig and lib/* via directory-relative - # paths; top-level zig/*.zig symlinks are no longer sufficient after the - # Zig 0.16 runtime/layout changes. - # - # The build dir stays per-process: it holds the generated - # ._clear_tmp_.zig, and concurrent or successive tests sharing one - # would read each other's source. + # Write to a minimal per-process build dir matching normal build layout. + # Transpiled tests import runtime/runtime-header.zig and lib/* via + # directory-relative paths; top-level zig/*.zig symlinks are no longer + # sufficient after the Zig 0.16 runtime/layout changes. base_name = File.basename(source, '.clear') build_dir = coverage_mode ? ZIG_DIR : File.join(ZIG_DIR, ".build-#{$$}") cleanup_paths = [] + ClearBuildSupport.reap_orphan_build_dirs!(ZIG_DIR) unless coverage_mode FileUtils.mkdir_p(build_dir) unless coverage_mode ClearBuildSupport.ensure_symlink(File.join(build_dir, 'runtime'), File.join(ZIG_DIR, 'runtime')) @@ -1542,6 +1628,9 @@ when 'test' cmd_parts += ['--global-cache-dir', File.join(shared_zig_cache, 'global')] cmd_parts += [tmp_name, 'runtime/switch.S', 'runtime/onRoot.S'] cmd_parts += ['-lc'] + # `--safe` / `--optimized` route through LLVM rather than the self-hosted + # backend, which is where the lexer keyword miscompile lived. + cmd_parts += ['-O', test_opt_level] unless test_opt_level == 'Debug' cmd_parts.concat(c_ffi_link_flags(c_libraries, build_dir, cleanup_paths)) if tag_filters.empty? cmd_parts += ['--test-filter', File.basename(source)] diff --git a/compiler/ruby/annotator/domains/control_flow.rb b/compiler/ruby/annotator/domains/control_flow.rb index 46925c17f..9f306e99a 100644 --- a/compiler/ruby/annotator/domains/control_flow.rb +++ b/compiler/ruby/annotator/domains/control_flow.rb @@ -487,8 +487,11 @@ def declare_is_a_binding!(condition) return unless payload_type scope = current_scope - scope.declare(binding, nil, payload_type, false, false, nil, :stack) - og_declare(binding, nil, payload_type) + # The IS_A node is the binding's declaration site. Recording it gives + # lowering a stable identity to key a rename on when a nested MATCH + # binds the same name. + scope.declare(binding, condition, payload_type, false, false, nil, :stack) + og_declare(binding, condition, payload_type) classify_ownership!(scope.local_entry!(binding)) borrow_match_payload_binding!(binding) return @@ -941,8 +944,8 @@ def declare_union_payload_binding!(node, match_case, plan, variant_name, binding end payload_type = match_payload_binding_type(plan, variant_name, T.unsafe(raw_payload), match_case) - current_scope.declare(binding, nil, payload_type, false, false, nil, :stack) - og_declare(binding, nil, payload_type) + current_scope.declare(binding, match_case, payload_type, false, false, nil, :stack) + og_declare(binding, match_case, payload_type) classify_ownership!(current_scope.local_entry!(binding)) borrow_match_payload_binding!(binding) unless node.takes end diff --git a/compiler/ruby/annotator/domains/lifetimes.rb b/compiler/ruby/annotator/domains/lifetimes.rb index 1748ab605..2348e1426 100644 --- a/compiler/ruby/annotator/domains/lifetimes.rb +++ b/compiler/ruby/annotator/domains/lifetimes.rb @@ -601,7 +601,13 @@ def handle_assign_borrow(node) error!(node, :BORROWED_VAR_NOT_FOUND) if borrowed_scope.nil? return if T.must(borrowed_scope).is_immutable?(root_var) - lhs_name = node.name.is_a?(AST::Identifier) ? node.name.name : "__borrow_#{root_var}" + # VarDecl#name is a String, Assignment#name an Identifier. Both are real + # bindings and must borrow under their own name; only a genuinely + # unbound result falls back to the synthetic name, whose lifetime + # nothing ever ends. + lhs_name = node.name + lhs_name = lhs_name.name if lhs_name.is_a?(AST::Identifier) + lhs_name = "__borrow_#{root_var}" unless lhs_name.is_a?(String) mutable = node.is_a?(AST::VarDecl) && node.mutable err = ownership_graph.borrow(lhs_name, root_var, mutable: mutable) error!(node, :LIFETIME_ALREADY_BORROWED, name: root_var) if err @@ -1296,7 +1302,7 @@ def cleanup_source_value(node) end private :cleanup_source_value - sig { params(name: String, node: T.nilable(AST::Node), type_info: Type::TypeInput).returns(T.nilable(T::Set[String])) } + sig { params(name: String, node: T.nilable(T.any(AST::Node, AST::MatchCase)), type_info: Type::TypeInput).returns(T.nilable(T::Set[String])) } def og_declare(name, node, type_info) T.bind(self, Annotator::Phases::TypeAnalysisSession) diff --git a/compiler/ruby/annotator/domains/member_access.rb b/compiler/ruby/annotator/domains/member_access.rb index aff56d8cf..0843d3fe1 100644 --- a/compiler/ruby/annotator/domains/member_access.rb +++ b/compiler/ruby/annotator/domains/member_access.rb @@ -380,7 +380,11 @@ def visit_HashLit(node) values = node.pairs.values if values.all? { |value| Type.new(value.resolved_type).string? } - value_type = :String + # Symbols are strings, but interned ones: collapsing them to a plain + # String drops @symbol and the map's values become owned slices that + # COPY deep-clones and cleanup frees. List literals already preserve + # the element capability. + value_type = values.all? { |value| value.full_type!(context: "hash literal symbol value").symbol? } ? :"String@symbol" : :String else value_type = values.first.resolved_type symbol_key_map = node.pairs.keys.all? { |key| key.is_a?(AST::Literal) && key.type == :SYMBOL } diff --git a/compiler/ruby/annotator/helpers/capabilities.rb b/compiler/ruby/annotator/helpers/capabilities.rb index 76fecaa98..d5fbf31e6 100644 --- a/compiler/ruby/annotator/helpers/capabilities.rb +++ b/compiler/ruby/annotator/helpers/capabilities.rb @@ -946,6 +946,12 @@ def declare_capability_scope!(fact) declare_unwrapped_capability_alias!(fact) if fact.unwraps_sync_alias? declare_capability_binding_or_error!(fact) declare_capability_projection!(fact) + # declare_with_new_capability marks the SOURCE binding, but the body reads + # through the alias and Scope#is_restricted? answers per binding. Without + # this the alias looks unrestricted, so borrowing through it -- e.g. calling + # a `RETURNS self: T` accessor -- is refused. + alias_name = fact.alias_name + current_scope.resolve_entry(alias_name)&.capabilities&.add(fact.capability) if alias_name nil end diff --git a/compiler/ruby/annotator/helpers/function_analysis.rb b/compiler/ruby/annotator/helpers/function_analysis.rb index 3b60140c7..65de36935 100644 --- a/compiler/ruby/annotator/helpers/function_analysis.rb +++ b/compiler/ruby/annotator/helpers/function_analysis.rb @@ -196,11 +196,30 @@ def analyze_routine(node, body, declared_return, is_implicit) return_type end + # The root scope also holds imported names and function entries; a routine + # body must only inherit the module's own variable declarations. + sig { returns(Scope) } + def module_variable_scope + T.bind(self, Annotator::Phases::TypeAnalysisSession) rescue nil + seeded = Scope.new + semantic_root_scope.binding_entries.each do |name, entry| + next unless entry.reg.is_a?(AST::VarDecl) + + seeded.binding_entries[name] = entry + end + seeded + end + sig { params(node: RoutineNode, blk: T.proc.void).void } def with_routine_analysis_scope(node, &blk) T.bind(self, Annotator::Phases::TypeAnalysisSession) rescue nil - with_new_scope do + # A routine body sees the module's own declarations: a module-level + # `MUTABLE x = ...` is Ruby module state, and `x = value` inside a function + # must reassign it. Seeding from the root scope (rather than a fresh one) + # is what makes the assignment resolve instead of silently declaring a + # shadowing local. + with_new_scope(module_variable_scope) do og_push_scope begin blk.call @@ -258,7 +277,7 @@ def signature_from_function_type(fn_type) name: "arg#{i}", type: param.type, required: true, - mutable: false, + mutable: param.mutable, takes: false ) i += 1 @@ -1287,7 +1306,11 @@ def verify_param_lifetime!(arg_node, param, signature) end return true unless base_paths.include?(:wildcard) || base_paths.include?(param.name) - error!(arg_node, :MUTABLE_PARAM_NEEDS_RESTRICT, name: param.name) + # FunctionSignature carries no name, so the diagnostic falls back to a + # generic label. (The former `respond_to?(:fn_name)` probe could never + # succeed here.) + error!(arg_node, :MUTABLE_PARAM_NEEDS_RESTRICT, + name: param.name, arg: arg_node.name, callee: "the callee") end # `node.return_lifetime` shapes: @@ -1558,7 +1581,15 @@ def declare_captures(node) nil, cap.storage ) - capture_entry.inherit_ownership_identity!(owner_entry) if owner_entry + next unless owner_entry + + capture_entry.inherit_ownership_identity!(owner_entry) + # A capture is the same binding seen from inside the lambda, so it keeps + # the source's capabilities. Without this, capturing a WITH alias -- + # `USE(MUTABLE view)` -- yields an entry with none, so + # Scope#is_restricted? is false for it and borrowing through the capture + # is refused. + capture_entry.capabilities.merge(owner_entry.capabilities) end nil end @@ -1670,12 +1701,15 @@ def return_is_borrow?(node) end if node.is_a?(AST::Identifier) return false unless ownership_graph[node.name]&.kind == :borrowed - # Parameters (reg=nil) and MATCH bindings (reg=nil) are safe to return — - # the caller controls their lifetime. Only flag variables explicitly assigned - # from a collection index borrow (BindExpr with container_borrow=true). + # Parameters and MATCH/IS_A payload bindings are safe to return — the + # caller controls their lifetime. (A payload binding records its MATCH + # arm as `reg` so lowering can rename a nested rebind of the same name; + # that node carries no container_borrow.) Only flag variables explicitly + # assigned from a collection index borrow (BindExpr with + # container_borrow=true). scope = lookup_scope_for(node.name) reg = scope&.resolve_entry(node.name)&.reg - return reg&.container_borrow == true + return !!(reg.respond_to?(:container_borrow) && reg.container_borrow == true) end return true if node.is_a?(AST::GetIndex) return true if node.is_a?(AST::GetField) diff --git a/compiler/ruby/annotator/helpers/function_return.rb b/compiler/ruby/annotator/helpers/function_return.rb index d3eaa028f..d85ed01bf 100644 --- a/compiler/ruby/annotator/helpers/function_return.rb +++ b/compiler/ruby/annotator/helpers/function_return.rb @@ -130,6 +130,11 @@ def resolve(receiver, args = []) element_list(value) when Kind::KeyList key = T.must(receiver).key_type + # keys() hands back the map's OWN keys, which it duplicated on insert and + # frees at deinit -- owned Strings, even when lookups are spelled with + # interned `String@symbol` handles. A Symbol is a handle nobody owns, so + # claiming one here would label map-owned bytes as immortal. + key = Type.new(:String) if key.symbol? element_list(key) when Kind::Infer resolve_infer(args) diff --git a/compiler/ruby/annotator/helpers/function_signature.rb b/compiler/ruby/annotator/helpers/function_signature.rb index 0029d26eb..48a6cc245 100644 --- a/compiler/ruby/annotator/helpers/function_signature.rb +++ b/compiler/ruby/annotator/helpers/function_signature.rb @@ -94,6 +94,11 @@ def initialize(params:, visibility: nil, type_params: [], reentrant: false, end end + # Stand-in for a registry `validate:` lambda while a signature is + # serialized. Procs cannot be marshalled, but every validator is a + # registry singleton, so the name is enough to re-link on load. + ValidatorRef = Struct.new(:name) + class AnalysisFacts < T::Struct extend T::Sig @@ -137,6 +142,29 @@ def copy return_def: return_def ) end + + sig { returns(T::Hash[Symbol, T.untyped]) } + def marshal_dump + state = T.let({}, T::Hash[Symbol, T.untyped]) + instance_variables.each { |ivar| state[ivar] = instance_variable_get(ivar) } + validator = state[:@arg_validator] + return state unless validator + + name = IntrinsicRegistry.validator_name(validator) + raise TypeError, "arg_validator is not a registry validator and cannot be serialized" unless name + + state[:@arg_validator] = ValidatorRef.new(name) + state + end + + sig { params(state: T::Hash[Symbol, T.untyped]).void } + def marshal_load(state) + state.each { |ivar, value| instance_variable_set(ivar, value) } + reference = @arg_validator + return unless reference.is_a?(ValidatorRef) + + @arg_validator = T.let(IntrinsicRegistry.validator_for_name(reference.name), T.nilable(Proc)) + end end private_constant :Contract, :AnalysisFacts @@ -723,6 +751,15 @@ def dup copy end + sig { params(entry: T.nilable(SymbolEntry)).returns(T.nilable(SymbolEntry)) } + def self.import_kept_identity_symbol(entry) + return nil unless entry&.kept_identity + + copy = entry.dup + copy.kept_identity = entry.kept_identity + copy + end + sig { params(params: T::Array[AST::Param]).returns(T::Array[AST::Param]) } def self.copy_params_for_import(params) params.map do |param| @@ -736,7 +773,10 @@ def self.copy_params_for_import(params) name_token: param.name_token, required: param.required, sync: param.sync, - symbol: nil + # The entry itself is mutable per-unit state, but kept_identity is a + # fact about the callee: drop it and an importer cannot tell the callee + # keeps the argument, so an Rc crosses the boundary without a retain. + symbol: import_kept_identity_symbol(param.symbol) ) end end diff --git a/compiler/ruby/annotator/helpers/generic_analysis.rb b/compiler/ruby/annotator/helpers/generic_analysis.rb index 6e8326885..762db5c3f 100644 --- a/compiler/ruby/annotator/helpers/generic_analysis.rb +++ b/compiler/ruby/annotator/helpers/generic_analysis.rb @@ -139,10 +139,20 @@ def type_annotation_facts(node, type_obj, is_param) sig { params(type_obj: Type).returns(Type) } def type_annotation_inner(type_obj) - return T.must(type_obj.payload_type) if type_obj.error_union? - return T.must(type_obj.wrapped_type) if type_obj.optional? + T.bind(self, Annotator::Phases::TypeAnalysisSession) + # Tense prefixes stack (`!?T`), so peel every layer -- a single unwrap + # leaves `!?String[]@set` looking like a non-array to the shape checks. + inner = type_obj + loop do + next_inner = if inner.error_union? + inner.payload_type + elsif inner.optional? + inner.wrapped_type + end + return inner unless next_inner - type_obj + inner = next_inner + end end sig { params(facts: TypeAnnotationFacts).void } diff --git a/compiler/ruby/annotator/helpers/intrinsic_registry.rb b/compiler/ruby/annotator/helpers/intrinsic_registry.rb index d42825df2..74b63fc86 100644 --- a/compiler/ruby/annotator/helpers/intrinsic_registry.rb +++ b/compiler/ruby/annotator/helpers/intrinsic_registry.rb @@ -44,6 +44,13 @@ module IntrinsicRegistry REGISTRY_VALUES = T.let({}, RegistryMap) MAP_METHOD_ALIASES_VALUE = T.let({}, T::Hash[String, String]) + # `validate:` lambdas are the only unserializable values a FunctionSignature + # carries, and every one of them is a registry singleton. Naming them lets a + # signature cross a process boundary (worker compiles, on-disk module cache) + # as a reference instead of a copy. + VALIDATORS_BY_NAME = T.let({}, T::Hash[String, Proc]) + VALIDATOR_NAMES = T.let({}, T::Hash[Integer, String]) + # Keys consumed at the FunctionSignature level (not IntrinsicEmit). FS_KEYS = %i[args arity validate return return_type can_fail error_fallible needs_rt].freeze @@ -504,10 +511,50 @@ def self.populate_registry_values MAP_METHOD_ALIASES.each do |key, value| MAP_METHOD_ALIASES_VALUE[key] = value end + name_validators! nil end private_class_method :populate_registry_values + sig { returns(NilClass) } + def self.name_validators! + REGISTRY_VALUES.each do |registry_name, registry| + registry.each do |key, entry| + entries = entry.is_a?(Array) ? entry : [entry] + entries.each_with_index do |raw, index| + next unless raw.is_a?(Hash) + + validator = raw[:validate] + next unless validator.is_a?(Proc) + + name = "#{registry_name}:#{registry_key_string(key)}:#{index}" + VALIDATORS_BY_NAME[name] = validator + VALIDATOR_NAMES[validator.object_id] = name + end + end + end + nil + end + private_class_method :name_validators! + + # Stable name for a registry `validate:` lambda, or nil when the Proc did + # not come from a registry (nothing else may cross a process boundary). + sig { params(validator: T.nilable(Proc)).returns(T.nilable(String)) } + def self.validator_name(validator) + return nil if validator.nil? + + registry_values + VALIDATOR_NAMES[validator.object_id] + end + + sig { params(name: T.nilable(String)).returns(T.nilable(Proc)) } + def self.validator_for_name(name) + return nil if name.nil? + + registry_values + VALIDATORS_BY_NAME[name] + end + # Idempotent normalizer for the flag-day migration: returns a # FunctionSignature for a registry/ad-hoc entry Hash, passes a # FunctionSignature through unchanged, and maps nil -> nil. Every diff --git a/compiler/ruby/annotator/phases/capability_audit_session.rb b/compiler/ruby/annotator/phases/capability_audit_session.rb index f9f5d7a8e..a8ef4ec04 100644 --- a/compiler/ruby/annotator/phases/capability_audit_session.rb +++ b/compiler/ruby/annotator/phases/capability_audit_session.rb @@ -65,6 +65,12 @@ def initialize(typed_program:, inputs:, source_code:, language_mode:, strict_tes language_mode: language_mode, strict_test: strict_test ), Context) + # Derived views of the frozen local_function_facts. Reentrance BFS + # indexes them once per queue step, so rebuilding per call is O(fns) + # inside an O(fns) walk. Keyed by the facts table they came from, so a + # republished TypedProgramFacts invalidates them. + @derived_call_views = T.let({}, T::Hash[Symbol, T::Hash[String, T::Set[String]]]) + @derived_call_views_source = T.let(nil, T.nilable(TypedProgramFacts::LocalFacts)) end sig { void } @@ -104,12 +110,30 @@ def function_body_summaries = @context.typed_program.body_summaries sig { returns(T::Hash[String, T::Set[String]]) } def function_call_graph - local_function_facts.transform_values { |function| function.callees.to_set } + derived_call_view(:callees) { |function| function.callees.to_set } end sig { returns(T::Hash[String, T::Set[String]]) } def function_propagating_callees - local_function_facts.transform_values { |function| function.propagating_callees.to_set } + derived_call_view(:propagating) { |function| function.propagating_callees.to_set } + end + + sig do + params(kind: Symbol, block: T.proc.params(arg0: LocalFunctionFacts).returns(T::Set[String])) + .returns(T::Hash[String, T::Set[String]]) + end + def derived_call_view(kind, &block) + facts = local_function_facts + unless @derived_call_views_source.equal?(facts) + @derived_call_views_source = facts + @derived_call_views.clear + end + cached = @derived_call_views[kind] + return cached if cached + + view = T.let({}, T::Hash[String, T::Set[String]]) + facts.each { |name, function| view[name] = block.call(function) } + @derived_call_views[kind] = view.freeze end sig { params(name: String).returns(T::Boolean) } diff --git a/compiler/ruby/annotator/phases/type_analysis_session.rb b/compiler/ruby/annotator/phases/type_analysis_session.rb index bb6636975..9561283d8 100644 --- a/compiler/ruby/annotator/phases/type_analysis_session.rb +++ b/compiler/ruby/annotator/phases/type_analysis_session.rb @@ -734,6 +734,25 @@ def validate_copy_linear_resource_facts!(program, facts) end private :validate_copy_linear_resource_facts! + # Signatures imported from another package carry their params (and the + # kept_identity stamped when that package was compiled), but their bodies are + # not in this unit's function registry. + sig { params(fn_nodes: T::Hash[String, AST::FunctionDef]).returns(T::Hash[String, T::Array[AST::Param]]) } + def imported_kept_params(fn_nodes) + out = T.let({}, T::Hash[String, T::Array[AST::Param]]) + semantic_root_scope.visible_entries.each do |name, entry| + key = name.to_s + next if fn_nodes.key?(key) + signature = entry.fn_signature + next unless signature + params = signature.params + next unless params.any? { |param| param.symbol&.kept_identity } + out[key] = params + end + out + end + private :imported_kept_params + sig { params(resolution: Annotator::Phases::ResolutionFacts).void } def apply_keep_analysis!(resolution) fn_nodes = resolution.function_registry.nodes @@ -742,6 +761,7 @@ def apply_keep_analysis!(resolution) EscapeAnalysis.apply_kept_identity_placement!( fn_nodes, body_summaries, + imported_params: imported_kept_params(fn_nodes), on_mutable_violation: lambda { |entry, arg, callee_name| sink = entry.kept_identity&.sink || fn_nodes[callee_name]&.params&.find { |p| p.symbol&.kept_identity }&.symbol&.kept_identity&.sink || diff --git a/compiler/ruby/annotator/protocol_projection_resolver.rb b/compiler/ruby/annotator/protocol_projection_resolver.rb index b71ceef55..4030b6149 100644 --- a/compiler/ruby/annotator/protocol_projection_resolver.rb +++ b/compiler/ruby/annotator/protocol_projection_resolver.rb @@ -141,7 +141,7 @@ def projection_protocol(projection, parameters, issues) result.dup end - sig { params(code: Symbol, values: T.untyped).returns(ProtocolProjectionIssue) } + sig { params(code: Symbol, values: T.any(Symbol, String)).returns(ProtocolProjectionIssue) } def issue(code, **values) arguments = T.let({}, T::Hash[Symbol, String]) values.each do |key, value| diff --git a/compiler/ruby/ast/ast.rb b/compiler/ruby/ast/ast.rb index aab298de1..29ee95705 100644 --- a/compiler/ruby/ast/ast.rb +++ b/compiler/ruby/ast/ast.rb @@ -37,6 +37,7 @@ module TensePlanValue # A static call carries the same call metadata: annotation copies it onto # the synthetic FuncCall it resolves through. AST::StaticCall, + AST::OptionalUnwrap, AST::TupleLit, AST::Cast, ) end BgNode = T.type_alias { T.any(AST::BgBlock, AST::BgStreamBlock) } @@ -148,6 +149,12 @@ def self.copy_pipeline_rewrite_metadata!(dst, src, include_call_metadata: false) dst.can_fail = src.can_fail unless src.can_fail.nil? dst.error_kind = src.error_kind if src.error_kind dst.error_type = src.error_type if src.error_type + # The importing module alias is what qualifies a cross-package call in + # the emitted Zig. Dropping it here emitted a bare callee that the + # package cannot see. + if src.is_a?(AST::FuncCall) && dst.is_a?(AST::FuncCall) && src.module_alias + dst.module_alias = src.module_alias + end end dst @@ -374,6 +381,15 @@ def predicate keyword_init: true) do extend T::Sig + # ruby-to-clear: field-type var_node=Locatable + # ruby-to-clear: field-type alias=?String + # ruby-to-clear: field-type alias_mutable=Bool + # ruby-to-clear: field-type guard_expr=?Locatable + # ruby-to-clear: field-type snapshot_token=?Token + # ruby-to-clear: field-type view_token=?Token + # ruby-to-clear: field-type view_length=?Locatable + # ruby-to-clear: field-type as_token=?Token + sig { params(kw: StructKwargs).void } def initialize(**kw) super @@ -2380,6 +2396,9 @@ def type=(val) extend T::Sig include Locatable # ruby-to-clear: field-type op=String@symbol + # ruby-to-clear: field-type left=Locatable + # ruby-to-clear: field-type right=Locatable + # ruby-to-clear: field-type paren_bind=?Bool # Derived: comparison/logical -> Bool; otherwise an operand's type. sig { returns(Type) } def full_type @@ -3000,6 +3019,8 @@ def wildcard?; field == '*' end def name; target.name end end GetIndex = Struct.new(:token, :target, :index) do + # ruby-to-clear: field-type index=Locatable + # ruby-to-clear: field-type target=Locatable extend T::Sig include Locatable attr_accessor :safe_nav_chain @@ -3790,6 +3811,10 @@ def child_bodies = branches.map(&:body) # Captured affine variables are MOVED into the fiber (not borrowed by pointer). # stack_size: :standard (default, 16 KB) | :micro (4 KB) | :large (64 KB) | :xl (256 KB) BgBlock = Struct.new(:token, :body, :deferred_drops, :stack_size, :pinned, :parallel, :arena_mode, :can_smash) do + # ruby-to-clear: field-type arena_mode=Bool + # ruby-to-clear: field-type can_smash=Bool + # ruby-to-clear: field-type parallel=Bool + # ruby-to-clear: field-type pinned=Bool extend T::Sig include Locatable include HasBodies @@ -3900,6 +3925,7 @@ def expr # case_drops: Array of drop-arrays (parallel to cases), filled by annotator # default_drops: drop-array for default branch (or nil), filled by annotator MatchStatement = Struct.new(:token, :expr, :cases, :default_case, :case_drops, :default_drops, :exhaustive, :takes) do + # ruby-to-clear: field-type exhaustive=Bool # ruby-to-clear: field-type expr=Locatable # ruby-to-clear: field-type cases=[]MatchCase # ruby-to-clear: field-type default_case=?([]Locatable) @@ -3938,6 +3964,10 @@ def child_bodies # ForRange: FOR var IN (start ..= end) DO body END # inclusive: true = ..= (start to end), false = ..< (start to end-1) ForRange = Struct.new(:token, :var_name, :start_expr, :end_expr, :inclusive, :body, :deferred_drops, :mark_per_iter, :tight) do + # ruby-to-clear: field-type start_expr=Locatable + # ruby-to-clear: field-type end_expr=Locatable + # ruby-to-clear: field-type inclusive=Bool + # ruby-to-clear: field-type body=[]Locatable extend T::Sig include Locatable include StatementVoidType @@ -4200,6 +4230,8 @@ def expression; self[:expression]; end # STUB fn RETURNS value | STUB fn CAPTURES var | STUB fn SEQUENCE [...] | STUB fn WITH lambda StubDecl = Struct.new(:token, :function_name, :kind, :value) { include Locatable } + # ruby-to-clear: field-type function_name=String + # ruby-to-clear: field-type kind=String@symbol # kind: :returns, :captures, :sequence, :with # ruby-to-clear: data-api diff --git a/compiler/ruby/ast/diagnostic_registry.rb b/compiler/ruby/ast/diagnostic_registry.rb index 01b8556ff..2e5cc8ca0 100644 --- a/compiler/ruby/ast/diagnostic_registry.rb +++ b/compiler/ruby/ast/diagnostic_registry.rb @@ -3328,8 +3328,11 @@ def self.entry(severity:, category:, template:, summary:, cause: nil, fix_hint: }, MUTABLE_PARAM_NEEDS_RESTRICT: { severity: :error, category: :lifetime, - template: "Lifetime Error: param `%{name}` is mutable, must be RESTRICTed before it can be borrowed.", - summary: "Mutable parameter must be RESTRICTed before being aliased.", + template: "Lifetime Error: cannot borrow through `%{arg}` -- it is mutable and not RESTRICTed. " \ + "`%{callee}` declares `RETURNS %{name}: T`, so its result borrows from the argument you pass as " \ + "`%{name}`. Wrap the read in `WITH RESTRICT %{arg} { ... }`, or bind an owned value with " \ + "`COPY %{callee}(%{arg})` if it must outlive a mutation of `%{arg}`.", + summary: "Borrowing through a mutable argument requires RESTRICT, or COPY to take ownership.", }, LIFETIME_RETURNS_REQUIRES_FAMILY_CONFLICT: { severity: :error, category: :lifetime, diff --git a/compiler/ruby/ast/fixable_suggestion_helper.rb b/compiler/ruby/ast/fixable_suggestion_helper.rb index d63d5ff9c..5d8cf0d50 100644 --- a/compiler/ruby/ast/fixable_suggestion_helper.rb +++ b/compiler/ruby/ast/fixable_suggestion_helper.rb @@ -25,11 +25,22 @@ def closest_name(input, candidates, max_distance: 3) best_distance <= max_distance ? best.to_s : nil end - sig { params(token: T.nilable(TypoToken), name: String, candidates: T::Array[String], message: String, fix_label: String, category: Symbol, cascade: T::Boolean).returns(NilClass) } + # The only thing a suggestion wants from its subject is where to point, and + # AnchorToken is exactly that pair. Narrowing to it here means the rest of + # the method never reaches through the union with T.unsafe. + sig { params(token: T.nilable(Lexer::Token)).returns(AnchorToken) } + def typo_anchor(token) + return AnchorToken.new(0, 0) if token.nil? + + AnchorToken.new(token.line, token.column) + end + + sig { params(token: T.nilable(Lexer::Token), name: String, candidates: T::Array[String], message: String, fix_label: String, category: Symbol, cascade: T::Boolean).returns(NilClass) } def emit_typo_suggestion!(token, name, candidates, message, fix_label, category: :registry, cascade: true) - token_line = T.cast(T.unsafe(token).line, Integer) - token_column = T.cast(T.unsafe(token).column, Integer) + anchor = typo_anchor(token) + token_line = anchor.line + token_column = anchor.column best = closest_name(name, candidates) fixes = T.let([], T::Array[Fix]) if best diff --git a/compiler/ruby/ast/lexer.rb b/compiler/ruby/ast/lexer.rb index 784e88b15..1858f6575 100644 --- a/compiler/ruby/ast/lexer.rb +++ b/compiler/ruby/ast/lexer.rb @@ -44,6 +44,32 @@ def text! raise TokenPayloadError, payload_error("text", "String") end + # Does this token's payload read as exactly `expected`? + # + # `token.value == "AS"` says the same thing only because Ruby lets a + # String-or-Integer-or-Float payload be compared to anything. A typed + # payload has to be narrowed to its String variant before the comparison + # means anything, and this is that narrowing, named once. + sig { params(expected: String).returns(T::Boolean) } + def text_is?(expected) + payload = value + return false unless payload.is_a?(String) + + payload == expected + end + + # consume_number yields an INT64 or a NUMBER token, and a count wants a + # whole number from either. `value.to_i` says that in Ruby only because the + # payload is untyped; naming the variants says it in both languages. + sig { returns(Integer) } + def number_as_integer + payload = value + return payload if payload.is_a?(Integer) + return payload.to_i if payload.is_a?(Float) + + raise TokenPayloadError, payload_error("number", "Integer or Float") + end + sig { returns(Integer) } def integer! payload = value diff --git a/compiler/ruby/ast/parser.rb b/compiler/ruby/ast/parser.rb index 16d274919..596274de3 100644 --- a/compiler/ruby/ast/parser.rb +++ b/compiler/ruby/ast/parser.rb @@ -144,8 +144,8 @@ class ParsedMatchArm < T::Struct AST::MinOp, AST::MaxOp, AST::AverageOp) end - @gradual_mode = T.let(false, T.nilable(T::Boolean)) - @ownership_mode = T.let(:default, T.nilable(Symbol)) + @gradual_mode = T.let(false, T::Boolean) + @ownership_mode = T.let(:default, Symbol) sig do params( @@ -207,23 +207,23 @@ class << self # build, one mode. sig { returns(T::Boolean) } def gradual_mode - T.must(@gradual_mode) + @gradual_mode end sig { params(value: T::Boolean).returns(T::Boolean) } def gradual_mode=(value) - @gradual_mode = T.let(value, T.nilable(T::Boolean)) + @gradual_mode = value value end sig { returns(Symbol) } def ownership_mode - @ownership_mode || :default + @ownership_mode end sig { params(value: Symbol).returns(Symbol) } def ownership_mode=(value) - @ownership_mode = T.let(value, T.nilable(Symbol)) + @ownership_mode = value value end @@ -244,7 +244,7 @@ def parse_type_syntax_document current_token = current unless current_token.type == :EOF error!(current_token, :PARSER_EXPECTED, - expected: "end of type", got: current_token.value, + expected: "end of type", got: current_token.display_value, type: current_token.type, line: current_token.line) end syntax diff --git a/compiler/ruby/ast/parser/collections_capabilities_and_tenses.rb b/compiler/ruby/ast/parser/collections_capabilities_and_tenses.rb index cb699380f..b51d418b5 100644 --- a/compiler/ruby/ast/parser/collections_capabilities_and_tenses.rb +++ b/compiler/ruby/ast/parser/collections_capabilities_and_tenses.rb @@ -13,7 +13,8 @@ def parse_capabilities result = CapabilityParseResult.new return result unless match?(:VAR_ID) && CAPABILITY_TOKENS.include?(current.value) - apply_capability!(result, consume(:VAR_ID)) + capability_token = consume(:VAR_ID) + apply_capability!(result, capability_token, capability_token.text!) # ':' chaining (e.g., @shared:locked, @soa:shared:locked, @list:soa) parse_capability_chain!(result) @@ -127,7 +128,7 @@ def apply_element_capability!(result, value) def token_char?(token, value) return false unless token - token.type == :CHAR && token.value == value + token.type == :CHAR && token.text_is?(value) end sig { params(token: T.nilable(Lexer::Token)).returns(T::Boolean) } @@ -137,7 +138,7 @@ def token_var?(token) # Apply a single capability token to the result hash. Detects duplicates. sig { params(result: CapabilityParseResult, token: Lexer::Token, value: String, validate_shard_count: T::Boolean).void } - def apply_capability!(result, token, value = token.value, validate_shard_count: false) + def apply_capability!(result, token, value, validate_shard_count: false) emit_boxed_capability_migration(token) ownership = CAPABILITY_OWNERSHIP_VALUES[value] if ownership @@ -176,7 +177,7 @@ def apply_capability!(result, token, value = token.value, validate_shard_count: error!(token, :DUPLICATE_SHARD_COUNT_CAP) if result.shard_count consume(:CHAR, '(') count_tok = consume_number - count = count_tok.value.to_i + count = count_tok.number_as_integer error!(count_tok, :SHARDED_TOO_FEW, count: count) if validate_shard_count && count < 2 result.shard_count = count consume(:CHAR, ')') @@ -246,15 +247,15 @@ def parse_with_capability if match?(:TYPE_ID) typo_tok = current emit_typo_suggestion!( - typo_tok, typo_tok.value, AST::CAPABILITIES.map(&:to_s), - "Unknown WITH capability '#{typo_tok.value}'", + typo_tok, typo_tok.display_value, AST::CAPABILITIES.map(&:to_s), + "Unknown WITH capability '#{typo_tok.display_value}'", "closest WITH capability", category: :capability, cascade: true ) end while match?(:KEYWORD) || match?(:VAR_ID) do - capability = if match?(:KEYWORD) && current.value != 'AS' + capability = if match?(:KEYWORD) && !current.text_is?('AS') cap_tok = consume(:KEYWORD) cap = cap_tok.text!.to_sym unless AST::CAPABILITIES.include?(cap) @@ -306,7 +307,7 @@ def parse_with_capability node.polymorphic = polymorphic if escape_tok node.deadlock_escape = { - kind: escape_tok.value == 'POSSIBLE_DEADLOCK' ? :deadlock : :lock_cycle, + kind: escape_tok.text_is?('POSSIBLE_DEADLOCK') ? :deadlock : :lock_cycle, token: escape_tok, } end @@ -321,7 +322,7 @@ def parse_with_capability node.polymorphic = polymorphic if escape_tok node.deadlock_escape = { - kind: escape_tok.value == 'POSSIBLE_DEADLOCK' ? :deadlock : :lock_cycle, + kind: escape_tok.text_is?('POSSIBLE_DEADLOCK') ? :deadlock : :lock_cycle, token: escape_tok, } end @@ -554,7 +555,7 @@ def match_optional_retry! return nil unless match!(:KEYWORD, 'RETRY') consume(:CHAR, '(') tok = consume_number - n = tok.value.to_i + n = tok.number_as_integer error!(tok, :RETRY_N_NONPOSITIVE, got: n) if n <= 0 consume(:CHAR, ')') consume(:KEYWORD, 'THEN') @@ -626,7 +627,7 @@ def parse_cap_join(tok, first_attrs) unless current.type == :VAR_ID error!(current, :EXPECTED_CAP_SIGIL_AFTER_COLON) end - normalized = current.value.start_with?('@') ? current.value : "@#{current.value}" + normalized = current.value.start_with?('@') ? current.value : "@#{current.display_value}" attrs = CAP_SIGIL_ATTRS[normalized] unless attrs # Chain form `@shared:foo` arrives without the `@`; root form @@ -636,7 +637,7 @@ def parse_cap_join(tok, first_attrs) candidates = has_at ? CAP_SIGIL_ATTRS.keys : CAP_SIGIL_ATTRS.keys.map { |k| k.sub(/^@/, '') } emit_typo_suggestion!( current, current.value, candidates, - "Unknown capability sigil '#{current.value}'", + "Unknown capability sigil '#{current.display_value}'", "closest capability sigil", category: :capability, cascade: true ) @@ -702,7 +703,7 @@ def parse_lock_rank_arg!(sigil_tok, attrs, dims) consume(:CHAR, ':') neg = match!(:CHAR, '-') num_tok = consume_number - rank = num_tok.value.to_i + rank = num_tok.number_as_integer rank = -rank if neg consume(:CHAR, ')') if dims.lock_rank @@ -856,6 +857,10 @@ def parse_bg_body_stmt rule = STMT_RULE_INDEX[ClearParser.token_rule_key(current)] return dispatch_stmt_rule(rule) if rule + # The chain is anchored where its first expression starts, which is the + # cursor right here -- reaching back through steps.first.expr.token asks + # an AST node union for a field instead. + chain_anchor = current parsed_var = current.type == :VAR_ID ? parse_var_form : nil if parsed_var&.assignment consume(:CHAR, ';') @@ -872,7 +877,7 @@ def parse_bg_body_stmt end unless match?(:KEYWORD, 'THEN') - error!(current, :EXPECTED_THEN_AFTER_AS_BG, got: current.value.inspect) + error!(current, :EXPECTED_THEN_AFTER_AS_BG, got: current.display_value.inspect) end steps = [AST::ThenStep.new(expr: expr, binding: binding_name)] @@ -887,7 +892,7 @@ def parse_bg_body_stmt steps << AST::ThenStep.new(expr: next_expr, binding: next_binding) end match!(:CHAR, ';') - return AST::ThenChain.new(steps.first.expr.token, steps) + return AST::ThenChain.new(chain_anchor, steps) end consume(:CHAR, ';') diff --git a/compiler/ruby/ast/parser/declarations_and_definitions.rb b/compiler/ruby/ast/parser/declarations_and_definitions.rb index 4afdc3ac5..0e1faf930 100644 --- a/compiler/ruby/ast/parser/declarations_and_definitions.rb +++ b/compiler/ruby/ast/parser/declarations_and_definitions.rb @@ -13,9 +13,9 @@ def parse_argument_specs # comptime: T — compile-time type parameter (EXTERN FN only) is_comptime = false - if match?(:VAR_ID) && current.value == "comptime" + if match?(:VAR_ID) && current.text_is?("comptime") # Peek ahead: if next is ':', it's a comptime param - if peek_at(1)&.type == :CHAR && peek_at(1)&.value == ":" + if peek_at(1)&.type == :CHAR && peek_at(1)&.text_is?(":") consume(:VAR_ID) # consume 'comptime' is_comptime = true end @@ -222,7 +222,7 @@ def parse_visibility_decl(visibility) elsif match?(:KEYWORD, 'CONST') parse_const_decl(visibility) else - error!(current, :VISIBILITY_BAD_KIND, got: current.value) + error!(current, :VISIBILITY_BAD_KIND, got: current.display_value) end end @@ -236,7 +236,7 @@ def parse_extern_decl elsif match?(:KEYWORD, 'STRUCT') parse_extern_struct(tok) else - error!(current, :EXTERN_BAD_KIND, got: current.value) + error!(current, :EXTERN_BAD_KIND, got: current.display_value) end end @@ -321,7 +321,7 @@ def parse_extern_return_lifetime return :wildcard end - return nil unless match?(:VAR_ID) && peek.type == :CHAR && peek.value == ':' + return nil unless match?(:VAR_ID) && peek.type == :CHAR && peek.text_is?(':') names = T.let([parse_var_id], T::Array[AST::Node]) consume(:CHAR, ':') @@ -412,7 +412,7 @@ def parse_extern_source(dependency, native_name) abi_token = current consume(abi_token.type) abi = abi_token.text!.downcase.to_sym - error!(abi_token, :PARSER_EXPECTED, expected: "C or ZIG", got: abi_token.value, + error!(abi_token, :PARSER_EXPECTED, expected: "C or ZIG", got: abi_token.display_value, type: abi_token.type, line: abi_token.line) unless %i[c zig].include?(abi) end if match!(:KEYWORD, 'CALLCONV') @@ -420,7 +420,7 @@ def parse_extern_source(dependency, native_name) consume(callconv_token.type) callconv = callconv_token.text!.downcase.to_sym error!(callconv_token, :PARSER_EXPECTED, expected: "C, SYSTEM, or WINAPI", - got: callconv_token.value, type: callconv_token.type, + got: callconv_token.display_value, type: callconv_token.type, line: callconv_token.line) unless %i[c system winapi].include?(callconv) end if match!(:KEYWORD, 'HEADER') @@ -510,8 +510,8 @@ def conformance_implementation_header? index = @pos while index < @tokens.length token = T.must(@tokens[index]) - return true if token.type == :KEYWORD && token.value == 'FOR' - return false if token.type == :CHAR && token.value == '{' + return true if token.type == :KEYWORD && token.text_is?('FOR') + return false if token.type == :CHAR && token.text_is?('{') index += 1 end false @@ -546,7 +546,7 @@ def parse_implementation_members else error!(current, :PARSER_EXPECTED, expected: "FN, METHOD, or } in IMPLEMENTATION", - got: current.value, type: current.type, line: current.line) + got: current.display_value, type: current.type, line: current.line) end stamp_source_range!(member, member_start, previous) members << member @@ -1213,7 +1213,7 @@ def starts_function_requirement? return true if match?(:KEYWORD, 'FN') return false unless match?(:KEYWORD, 'PUB') || match?(:KEYWORD, 'PRIVATE') - peek.type == :KEYWORD && peek.value == 'FN' + peek.type == :KEYWORD && peek.text_is?('FN') end sig { returns(T::Array[Symbol]) } @@ -1304,7 +1304,7 @@ def parse_let_binding # the test-block / when-block parsers; both share the same hook syntax. sig { params(first: String, second: String).returns(T::Boolean) } def test_hook_match?(first, second) - match?(:KEYWORD, first) && @tokens[@pos + 1]&.value == second + match?(:KEYWORD, first) && @tokens[@pos + 1]&.text_is?(second) == true end # Parse `BEFORE EACH DO END` (or AFTER EACH); returns the body @@ -1335,11 +1335,11 @@ def parse_when_block lets = [] until match?(:KEYWORD, 'END') - if match?(:KEYWORD, 'TEST') && @tokens[@pos + 1]&.value == 'THAT' + if match?(:KEYWORD, 'TEST') && @tokens[@pos + 1]&.text_is?('THAT') tests << parse_test_that elsif match?(:KEYWORD, 'PENDING') && - @tokens[@pos + 1]&.value == 'TEST' && - @tokens[@pos + 2]&.value == 'THAT' + @tokens[@pos + 1]&.text_is?('TEST') && + @tokens[@pos + 2]&.text_is?('THAT') # PENDING TEST THAT "..." DO ... END — type-checked but skipped # at runtime via `return error.SkipZigTest;` in lowering. consume(:KEYWORD, 'PENDING') @@ -1420,7 +1420,7 @@ def parse_assert_raises # Peek: if next is TYPE_ID followed by comma, it's ASSERT_RAISES Kind, ErrorName, expr error_name = nil - if current.type == :TYPE_ID && @tokens[@pos + 1]&.type == :CHAR && @tokens[@pos + 1]&.value == ',' + if current.type == :TYPE_ID && @tokens[@pos + 1]&.type == :CHAR && @tokens[@pos + 1]&.text_is?(',') error_name = consume(:TYPE_ID).text! consume(:CHAR, ',') end @@ -1439,8 +1439,11 @@ def parse_benchmark_stmt # Parse optional iteration count: x1000 or x 1000 iterations = 1000 # default - if match?(:VAR_ID) && current.value =~ /^x(\d+)$/ - iterations = $1.to_i + # Read the count off the token rather than through $~, which is global + # match state the self-hosted parser has no equivalent for. + count_text = current.display_value + if match?(:VAR_ID) && count_text.match?(/\Ax\d+\z/) + iterations = count_text[1..].to_i consume(:VAR_ID) end consume(:CHAR, ';') diff --git a/compiler/ruby/ast/parser/expressions_and_postfix.rb b/compiler/ruby/ast/parser/expressions_and_postfix.rb index d950663de..62072c4be 100644 --- a/compiler/ruby/ast/parser/expressions_and_postfix.rb +++ b/compiler/ruby/ast/parser/expressions_and_postfix.rb @@ -96,7 +96,7 @@ def reject_legacy_select_effect_spelling! return unless match?(:CHAR, '!') || match?(:CHAR, '?') marker = current.value - marker += '?' if marker == '!' && peek.type == :CHAR && peek.value == '?' + marker += '?' if marker == '!' && peek.type == :CHAR && peek.text_is?('?') fix = Fix.new( description: fix_description(:INSERT_SELECT_EFFECT_COLON, selector: "SELECT:#{marker}"), confidence: :auto, @@ -270,7 +270,7 @@ def suffix_rule_applicable?(rule, lhs) sig { returns(T::Boolean) } def conditional_binding_suffix? - peek.type == :KEYWORD && peek.value == 'AS' + peek.type == :KEYWORD && peek.text_is?('AS') end sig { params(lhs: AST::Node).returns(AST::UnaryOp) } @@ -346,11 +346,11 @@ def parse_dot_suffix(lhs) # Join only this exact positional spelling; ordinary names cannot # absorb a trailing number here. if name == "_" && (match?(:NUMBER) || match?(:INT64)) - name = "_#{consume_number.value}" + name = "_#{consume_number.display_value}" end # Predicate suffix: name? followed by ( → method call with ? suffix - if match?(:CHAR, '?') && peek_at(1)&.value == '(' + if match?(:CHAR, '?') && peek_at(1)&.text_is?('(') consume(:CHAR, '?') name = "#{name}?" end @@ -359,7 +359,8 @@ def parse_dot_suffix(lhs) # Method Call _, args = parse_comma_seq(:CHAR, '(', ')') { parse_expression } call = AST::MethodCall.new(name_token, lhs, name, args) - stamp_source_range_from_node!(call, lhs, previous) + call.source_range = source_range_from_node(lhs, previous) + call else # Field Access AST::GetField.new(name_token, lhs, name) @@ -371,7 +372,7 @@ def parse_dot_suffix(lhs) def parse_func_call_suffix(lhs) start_token, args = parse_comma_seq(:CHAR, '(', ')') { parse_expression } call = AST::FuncCall.new(start_token, lhs, args) - stamp_source_range_from_node!(call, lhs, previous) + call.source_range = source_range_from_node(lhs, previous) call end @@ -429,7 +430,7 @@ def tense_navigation_marker_run offset += 1 end dot = peek_at(offset) - return nil if markers.empty? || dot.nil? || dot.type != :CHAR || dot.value != "." + return nil if markers.empty? || dot.nil? || dot.type != :CHAR || !dot.text_is?(".") markers end @@ -606,9 +607,12 @@ def parse_binary_op(lhs, op_token, op_prec) case op_val when 'AS' + # Anchor on the cursor before the operand: reaching for as_rhs.token asks + # an AST node union for a field. + as_anchor = current as_rhs = parse_var_id unless as_rhs.is_a?(AST::Identifier) - error!(as_rhs, :EXPECTED_IDENT_AFTER_AS, got: "expression") + error!(as_anchor, :EXPECTED_IDENT_AFTER_AS, got: "expression") end return AST::BinaryOp.new(op_token, lhs, :BIND_VAR, as_rhs) @@ -782,12 +786,12 @@ def parse_unary end # Call-site override syntax is reserved here; the annotator rejects it # until runtime semantics are implemented. - if current.type == :VAR_ID && (current.value == '@thunk' || current.value == '@maxDepth') + if current.type == :VAR_ID && (current.text_is?('@thunk') || current.text_is?('@maxDepth')) sigil_tok = consume(:VAR_ID) consume(:CHAR, '(') n_tok = current n_lit = consume_number - n = n_lit.value.to_i + n = n_lit.number_as_integer if n <= 0 error!(n_tok, :SIGIL_N_NONPOSITIVE, sigil: sigil_tok.text!, count: n) end @@ -835,7 +839,7 @@ def parse_var_id node = T.let(AST::Identifier.new(var_token, name), AST::Node) # Predicate suffix: name? followed by ( → function call with ? suffix - if match?(:CHAR, '?') && peek_at(1)&.value == '(' + if match?(:CHAR, '?') && peek_at(1)&.text_is?('(') consume(:CHAR, '?') name = "#{name}?" end @@ -855,10 +859,10 @@ def parse_primary rule = PRIMARY_RULE_INDEX[ClearParser.token_rule_key(current)] rule ||= PRIMARY_RULE_INDEX[ClearParser.rule_key(current.type, nil)] return dispatch_primary_rule(rule) if rule - return parse_unary() if current.type == :CHAR && (AST::UNARY_OPS.include?(current.value) || current.value == '&') + return parse_unary() if current.type == :CHAR && (AST::UNARY_OPS.include?(current.value) || current.text_is?('&')) lit = parse_lit(:stack) return parse_suffixes(lit) if !lit.nil? - error!(current, :UNEXPECTED_TOKEN_LINE, value: current.value, type: current.type, line: current.line) + error!(current, :UNEXPECTED_TOKEN_LINE, value: current.display_value, type: current.type, line: current.line) end # Returns true if, starting from current position '<', the token stream matches @@ -873,10 +877,10 @@ def peek_generic_angle_params?(end_char) loop do token = peek_at(offset) return false unless token - if token.type == :CHAR && token.value == '<' + if token.type == :CHAR && token.text_is?('<') depth += 1 - elsif token.type == :CHAR && (token.value == '>' || token.value == '>>') - depth -= token.value == '>>' ? 2 : 1 + elsif token.type == :CHAR && (token.text_is?('>') || token.text_is?('>>')) + depth -= token.text_is?('>>') ? 2 : 1 if depth == 0 following = peek_at(offset + 1) return !following.nil? && following.type == :CHAR && following.value == end_char @@ -1044,7 +1048,7 @@ def parse_window_op window_token = consume(:KEYWORD, 'WINDOW') consume(:CHAR, '(') # Named-param form (BatchWindowOp) if first token is VAR_ID followed by ':' - if match?(:VAR_ID) && peek.type == :CHAR && peek.value == ':' + if match?(:VAR_ID) && peek.type == :CHAR && peek.text_is?(':') options = {} loop do key_tok = consume(:VAR_ID) @@ -1144,7 +1148,7 @@ def parse_concurrent_inner_op(parent_token) expr = parse_expression(1) AST::AverageOp.new(previous, expr) else - error!(current, :CONCURRENT_BAD_OP, got: current.value.inspect) + error!(current, :CONCURRENT_BAD_OP, got: current.display_value.inspect) end end @@ -1157,8 +1161,10 @@ def parse_each_op parse_brace_block else callback = parse_expression(1) - [AST::FuncCall.new(token, callback.respond_to?(:name) ? T.unsafe(callback).name : callback.to_s, - [AST::Identifier.new(token, "_")])] + # An Identifier callback names the function; anything else stands for + # itself. respond_to? here would make the subject the whole node union. + callee = callback.is_a?(AST::Identifier) ? callback.name : callback.to_s + [AST::FuncCall.new(token, callee, [AST::Identifier.new(token, "_")])] end AST::EachOp.new(token, body) end @@ -1174,7 +1180,8 @@ def parse_tap_op else # Short form: TAP func -> becomes TAP { func(_); } expr = parse_expression(1) # parse_pipe_expression - AST::TapOp.new(token, [AST::FuncCall.new(token, expr.respond_to?(:name) ? T.unsafe(expr).name : expr.to_s, [AST::Identifier.new(token, "_")])]) + callee = expr.is_a?(AST::Identifier) ? expr.name : expr.to_s + AST::TapOp.new(token, [AST::FuncCall.new(token, callee, [AST::Identifier.new(token, "_")])]) end end diff --git a/compiler/ruby/ast/parser/predicates_and_refinements.rb b/compiler/ruby/ast/parser/predicates_and_refinements.rb index d43f11aa2..8af8a423f 100644 --- a/compiler/ruby/ast/parser/predicates_and_refinements.rb +++ b/compiler/ruby/ast/parser/predicates_and_refinements.rb @@ -11,7 +11,7 @@ class ClearParser def parse_comptime_statement consume(:KEYWORD, 'COMPTIME') unless match?(:KEYWORD, 'IF') - error!(current, :PARSER_EXPECTED, expected: "IF", got: current.value, type: current.type, line: current.line) + error!(current, :PARSER_EXPECTED, expected: "IF", got: current.display_value, type: current.type, line: current.line) end parse_if_statement(is_comptime: true) end @@ -94,7 +94,7 @@ def parse_refined_if_chain(if_token, is_comptime: false) break unless match?(:KEYWORD, 'AND') || match?(:KEYWORD, 'OR') operator = consume(:KEYWORD) - error!(operator, :CONDITIONAL_BINDING_UNDER_OR) if operator.value == 'OR' + error!(operator, :CONDITIONAL_BINDING_UNDER_OR) if operator.text_is?('OR') end if match?(:ARROW, '->') @@ -134,7 +134,7 @@ def conditional_capture_ahead? return false if depth == 0 && ((token.type == :KEYWORD && %w[THEN ELSE END].include?(token.value)) || token.type == :ARROW || token.type == :EOF) if token.type == :KEYWORD && %w[EXISTS IS_OK].include?(token.value) following = peek_at(offset + 1) - return true if following && following.type == :KEYWORD && following.value == 'AS' + return true if following && following.type == :KEYWORD && following.text_is?('AS') end offset += 1 end @@ -145,7 +145,7 @@ def refinement_steps(node, if_token) return [node] unless node.is_a?(AST::BinaryOp) if node.op == :BIND_VAR right = T.cast(node.right, AST::Identifier) - predicate = node.token.value == 'IS_OK' ? :is_ok : :exists + predicate = node.token.text_is?('IS_OK') ? :is_ok : :exists return [AST::Binding.new(expr: node.left, name: right.name, name_token: right.token, predicate: predicate)] end if node.op == :OR && contains_refinement_binding?(node) @@ -205,7 +205,7 @@ def conditional_binding_predicate? sig { params(expr: AST::Node).returns(AST::Binding) } def parse_conditional_binding(expr) predicate_tok = consume(:KEYWORD) - predicate = predicate_tok.value == 'IS_OK' ? :is_ok : :exists + predicate = predicate_tok.text_is?('IS_OK') ? :is_ok : :exists consume(:KEYWORD, 'AS') name_tok = consume(:VAR_ID) AST::Binding.new(expr: expr, name: name_tok.text!, name_token: name_tok, predicate: predicate) diff --git a/compiler/ruby/ast/parser/state.rb b/compiler/ruby/ast/parser/state.rb index 3fe117fcf..0c7b6c694 100644 --- a/compiler/ruby/ast/parser/state.rb +++ b/compiler/ruby/ast/parser/state.rb @@ -9,6 +9,21 @@ def parser_error_host? true end + # The parser anchors every diagnostic on a token it already holds, never on + # an AST node, so it does not need ErrorHelper's `respond_to?(:token)` walk. + # Saying so is what lets the self-hosted parser type this surface at all: a + # duck-typed subject would have to be the whole AST::Locatable union, and + # reading a field off a union is not something CLEAR can express. + # Parameter types stay as wide as ErrorHelper's -- sorbet-runtime requires + # an override to be contravariant -- but the return types are narrowed to + # the token the parser actually always has, which is what the self-hosted + # signatures are written against. + sig { params(node_or_token: T.untyped).returns(T.nilable(Lexer::Token)) } + def diagnostic_token(node_or_token) = node_or_token + + sig { params(token: DiagnosticToken).returns(T.nilable(Lexer::Token)) } + def source_error_token(token) = T.cast(token, T.nilable(Lexer::Token)) + include ErrorHelper SYNTAX_TOKENS_AT_STATEMENT_END = T.let( @@ -18,6 +33,10 @@ def parser_error_host? # Partial-class files are compiled as separate CLEAR packages during # self-hosting, so restate the storage types they read from parser.rb. + # ruby-to-clear: field-type budget=FrontendResourceBudget@multiowned + # ruby-to-clear: field-type wrapper_operand_precedence=?Int64 + # ruby-to-clear: field-type delimiter_closings=[]?Int64 + # ruby-to-clear: field-type gradual=Bool # ruby-to-clear: field-type pos=Int64 # ruby-to-clear: field-type source_code=String # ruby-to-clear: field-type tokens=[]Token @@ -85,7 +104,7 @@ def consume_number @pos += 1 tok else - error!(current, :EXPECTED_NUMBER, value: current.value, type: current.type) + error!(current, :EXPECTED_NUMBER, value: current.display_value, type: current.type) end end @@ -153,7 +172,7 @@ def emit_consume_error_with_fix(token, expected_type, expected_value) error!(token, :LEGACY_MUTATION_NAME_SUFFIX) end - error!(token, :PARSER_EXPECTED, expected: expected_value || expected_type, got: token.value, type: token.type, line: token.line) + error!(token, :PARSER_EXPECTED, expected: expected_value || expected_type, got: token.display_value, type: token.type, line: token.line) end # Insert `` at the end of the previous source line (right @@ -178,7 +197,7 @@ def emit_syntax_insert_end_of_line!(prev_tok, next_tok, expected_value) code: :PARSER_EXPECTED_AT_END_OF_LINE, expected: expected_value, expected_line: prev_tok.line, - got: next_tok.value, + got: next_tok.display_value, got_line: next_tok.line, category: :type, level: :error, fixes: [fix], raise_in_collector: true) @@ -200,7 +219,7 @@ def emit_syntax_insert_before_token!(token, expected_value) fixable!(token, code: :PARSER_EXPECTED_BEFORE_TOKEN, expected: expected_value, - got: token.value, + got: token.display_value, line: token.line, category: :type, level: :error, fixes: [fix], raise_in_collector: true) @@ -300,13 +319,16 @@ def stamp_source_range!(node, first, last) # an expression. The token carried by a MethodCall is the method name, not # the beginning of `receiver.method(...)`; diagnostics and source rewrites # need the latter. - sig { params(node: AST::Node, first: AST::Locatable, last: Lexer::Token).returns(AST::Node) } - def stamp_source_range_from_node!(node, first, last) + # Returns the range rather than stamping it: the caller holds the concrete + # node and can assign the field directly, which is one narrowing instead of + # one per node type. + sig { params(first: AST::Locatable, last: Lexer::Token).returns(AST::SourceRange) } + def source_range_from_node(first, last) source_range = first.source_range raise "Internal: source range missing from postfix receiver" unless source_range range = source_range end_offset = last.end_offset || ((last.start_offset || range.end_offset) + last.value.to_s.bytesize) - node.source_range = AST::SourceRange.new( + AST::SourceRange.new( file: range.file || last.file, start_offset: range.start_offset, end_offset: end_offset, @@ -315,6 +337,5 @@ def stamp_source_range_from_node!(node, first, last) end_line: last.end_line || last.line, end_column: last.end_column || (last.column + last.value.to_s.length), ) - node end end diff --git a/compiler/ruby/ast/parser/statements_and_control_flow.rb b/compiler/ruby/ast/parser/statements_and_control_flow.rb index 347466136..240d0a5aa 100644 --- a/compiler/ruby/ast/parser/statements_and_control_flow.rb +++ b/compiler/ruby/ast/parser/statements_and_control_flow.rb @@ -91,29 +91,29 @@ def parse_defer sig { params(token: Lexer::Token, body: T::Array[AST::Node]).void } def reject_defer_control_flow!(token, body) - stack = T.let(body.dup, T::Array[AST::Node]) - until stack.empty? - node = stack.pop - next unless node.is_a?(AST::Locatable) - if node.is_a?(AST::ReturnNode) || node.is_a?(AST::BreakNode) || - node.is_a?(AST::ContinueNode) || node.is_a?(AST::YieldExpr) - kind = node.class.name.to_s.split("::").last - error!(token, :DEFER_NO_CONTROL_FLOW, kind: kind) - end - # FN/lambda bodies are their own control-flow scopes. - next if node.is_a?(AST::FunctionDef) || node.is_a?(AST::LambdaLit) - - node.class.members.each do |member| - value = T.unsafe(node)[member] - if value.is_a?(Array) - value.each { |child| stack << child if child.is_a?(AST::Locatable) } - elsif value.is_a?(AST::Locatable) - stack << value - end + body.each do |root| + # each_locatable already stops at FN/lambda bodies, which are their own + # control-flow scopes, and walks children without Struct reflection. + AST.each_locatable(root) do |node| + kind = defer_control_flow_kind(node) + error!(token, :DEFER_NO_CONTROL_FLOW, kind: kind) if kind end end end + # The node class name the diagnostic wants, for exactly the four kinds DEFER + # rejects. `node.class.name` says this by reflection, which does not survive + # translation and cannot be checked. + sig { params(node: AST::Node).returns(T.nilable(String)) } + def defer_control_flow_kind(node) + return "ReturnNode" if node.is_a?(AST::ReturnNode) + return "BreakNode" if node.is_a?(AST::BreakNode) + return "ContinueNode" if node.is_a?(AST::ContinueNode) + return "YieldExpr" if node.is_a?(AST::YieldExpr) + + nil + end + sig { returns(AST::BreakNode) } def parse_break token = consume(:KEYWORD, 'BREAK') @@ -314,14 +314,14 @@ def parse_inferred_wrapper_annotation start = current.value next_token = peek_at(1) - suffix = if start == '!' && next_token&.type == :CHAR && T.must(next_token).value == '?' + suffix = if start == '!' && next_token&.type == :CHAR && T.must(next_token).text_is?('?') "!?".freeze else start end required_count = suffix.length terminal = peek_at(required_count) - return nil unless terminal&.type == :CHAR && T.must(terminal).value == '=' + return nil unless terminal&.type == :CHAR && T.must(terminal).text_is?('=') required_count.times { consume(:CHAR) } Type.new("#{suffix}Auto") @@ -465,7 +465,7 @@ def parse_value_block_expr end unless result - error!(current, :UNEXPECTED_TOKEN_LINE, value: current.value, type: current.type, line: current.line) + error!(current, :UNEXPECTED_TOKEN_LINE, value: current.display_value, type: current.type, line: current.line) end consume(:CHAR, '}') @@ -721,7 +721,7 @@ def parse_struct_pattern if match?(:CHAR, ':') consume(:CHAR, ':') # `_` as value means wildcard — ignore this field's value - if current.type == :VAR_ID && current.value == '_' + if current.type == :VAR_ID && current.text_is?('_') consume(:VAR_ID) fields << AST::PatternField.new(name: name, value: :wildcard, name_token: name_tok) else @@ -767,7 +767,8 @@ def parse_catch_item # Parse a single CATCH WITH filter: a TYPE_ID (error type) or a # STRING literal (message). - sig { returns(T.nilable(AST::CatchFilter)) } + # Every branch either builds a filter or raises, so this never yields nil. + sig { returns(AST::CatchFilter) } def parse_catch_filter if match?(:TYPE_ID) tok = consume(:TYPE_ID) diff --git a/compiler/ruby/ast/parser/types.rb b/compiler/ruby/ast/parser/types.rb index 2f0b77e90..a1b0040ac 100644 --- a/compiler/ruby/ast/parser/types.rb +++ b/compiler/ruby/ast/parser/types.rb @@ -15,9 +15,12 @@ def parse_fn_type_annotation consume(:KEYWORD, 'FN') consume(:CHAR, '(') param_types = [] + param_mutability = [] until match?(:CHAR, ')') + # A callback parameter the callee may mutate: FN(MUTABLE T) -> R. + param_mutability << match!(:KEYWORD, 'MUTABLE') # Allow optional name annotation: `name: Type` or just `Type` - if match?(:VAR_ID) && peek.type == :CHAR && peek.value == ':' + if match?(:VAR_ID) && peek.type == :CHAR && peek.text_is?(':') consume(:VAR_ID) # name is for documentation only consume(:CHAR, ':') end @@ -32,13 +35,14 @@ def parse_fn_type_annotation abi_token = current consume(abi_token.type) abi = abi_token.text!.downcase.to_sym - error!(abi_token, :PARSER_EXPECTED, expected: "C", got: abi_token.value, + error!(abi_token, :PARSER_EXPECTED, expected: "C", got: abi_token.display_value, type: abi_token.type, line: abi_token.line) unless abi == :c end if match?(:VAR_ID) && %w[@reentrant @nonReentrant].include?(current.value) - error!(current, :PARSER_EXPECTED, expected: "supported function type annotation", got: current.value, type: current.type, line: current.line) + error!(current, :PARSER_EXPECTED, expected: "supported function type annotation", got: current.display_value, type: current.type, line: current.line) end - Type.function_type_from_parts(param_types, T.unsafe(return_type), false, nil, abi) + Type.function_type_from_parts(param_types, T.unsafe(return_type), false, nil, abi, + param_mutability.map { |flag| !!flag }) end sig { params(migration_root: T::Boolean).returns(Type) } @@ -104,7 +108,7 @@ def parse_type_annotation_body # of optional T while ?(T[]) means an optional list of T. if optional_prefix == "?" && match?(:CHAR, '(') if tense_prefix != "" || error_prefix != "" - error!(current, :PARSER_EXPECTED, expected: "a grouped optional without an outer tense/error prefix", got: current.value, type: current.type, line: current.line) + error!(current, :PARSER_EXPECTED, expected: "a grouped optional without an outer tense/error prefix", got: current.display_value, type: current.type, line: current.line) end consume(:CHAR, '(') wrapped = parse_type_annotation(migration_root: false) @@ -190,7 +194,7 @@ def parse_type_annotation_body # Case 3: Fixed Explicit "Number[10]" elsif match?(:NUMBER) || match?(:INT64) - size = consume_number.value.to_i + size = consume_number.number_as_integer consume(:CHAR, ']') inner = "[#{size}]" @@ -202,7 +206,7 @@ def parse_type_annotation_body got: "[?]", type: current.type, line: current.line) # Case 5: Infinite stream marker "T[INF]" (used inside tense type ~T[INF]) - elsif match?(:TYPE_ID) && current.value == 'INF' + elsif match?(:TYPE_ID) && current.text_is?('INF') consume(:TYPE_ID) consume(:CHAR, ']') inner = "[INF]" @@ -217,7 +221,7 @@ def parse_type_annotation_body if match!(:CHAR, ']') inner += "[]" elsif match?(:NUMBER) || match?(:INT64) - size = consume_number.value.to_i + size = consume_number.number_as_integer consume(:CHAR, ']') inner += "[#{size}]" else @@ -350,11 +354,11 @@ def parse_inline_type_expression token = T.must(peek_at(0)) if token.type == :CHAR return parse_inline_prefixed_expression if %w[? ! ~].include?(token.value) - if token.value == '[' - return parse_inline_stream_expression if peek_at(1)&.value == '~' + if token.text_is?('[') + return parse_inline_stream_expression if peek_at(1)&.text_is?('~') return parse_inline_linear_expression end - return parse_inline_map_expression if token.value == '{' + return parse_inline_map_expression if token.text_is?('{') end parse_inline_atom_expression @@ -366,8 +370,8 @@ def parse_inline_stream_expression consume(:CHAR, '~') cardinality = T.let(:FINITE, T.any(Integer, Symbol)) if match?(:NUMBER) || match?(:INT64) - cardinality = consume_number.value.to_i - elsif match?(:TYPE_ID) && current.value == "INF" + cardinality = consume_number.number_as_integer + elsif match?(:TYPE_ID) && current.text_is?("INF") consume(:TYPE_ID, 'INF') cardinality = :INF end @@ -390,11 +394,11 @@ def parse_inline_prefixed_expression expected: "an optional stream item such as [~]?T", got: "?[~]T", type: prefix_token.type, line: prefix_token.line) end - return TypeExpression.of(OptionalTypeExpression.new(inner: inner)) + return TypeExpression.of(OptionalTypeExpression.new(inner: inner), inner.capabilities) end - return TypeExpression.of(FallibleTypeExpression.new(inner: inner)) if prefix == "!" + return TypeExpression.of(FallibleTypeExpression.new(inner: inner), inner.capabilities) if prefix == "!" - TypeExpression.of(FutureTypeExpression.new(inner: inner)) + TypeExpression.of(FutureTypeExpression.new(inner: inner), inner.capabilities) end sig { returns(TypeExpression) } @@ -443,7 +447,7 @@ def parse_inline_linear_expression dimensions = T.let([], T::Array[T.any(Integer, Symbol)]) allocation_hint = T.let(nil, T.nilable(Integer)) if match?(:NUMBER) || match?(:INT64) - dimension = consume_number.value.to_i + dimension = consume_number.number_as_integer else layout = consume(:TYPE_ID).text! case layout @@ -451,25 +455,25 @@ def parse_inline_linear_expression kind = layout.downcase.to_sym dimension = layout == "List" ? :LIST : :SET if match!(:CHAR, '(') - allocation_hint = consume_number.value.to_i + allocation_hint = consume_number.number_as_integer consume(:CHAR, ')') end when "Pool" kind = :pool consume(:CHAR, '(') - dimension = consume_number.value.to_i + dimension = consume_number.number_as_integer consume(:CHAR, ')') else error!(previous, :PARSER_EXPECTED, expected: "an Inline Pivot dimension", got: layout, type: previous.type, line: previous.line) end end - dimensions << T.must(dimension) + dimensions << dimension while match!(:CHAR, ',') if !allocation_hint.nil? || kind == :set || kind == :pool - error!(previous, :PARSER_EXPECTED, expected: "integer or List dimensions in a flat rank", got: previous.value, type: previous.type, line: previous.line) + error!(previous, :PARSER_EXPECTED, expected: "integer or List dimensions in a flat rank", got: previous.display_value, type: previous.type, line: previous.line) end if match?(:NUMBER) || match?(:INT64) - dimensions << consume_number.value.to_i + dimensions << consume_number.number_as_integer else layout = consume(:TYPE_ID).text! unless layout == "List" @@ -498,7 +502,7 @@ def parse_inline_map_expression else parsed_key = parse_inline_type_expression if match?(:CHAR, ',') - error!(current, :PARSER_EXPECTED, expected: "a closing brace; nested maps use separate brace layers", got: current.value, type: current.type, line: current.line) + error!(current, :PARSER_EXPECTED, expected: "a closing brace; nested maps use separate brace layers", got: current.display_value, type: current.type, line: current.line) end parsed_key end @@ -513,7 +517,7 @@ def parse_inline_capabilities(collection: nil) unless parsed.collection.nil? error!(previous, :PARSER_EXPECTED, expected: "collection topology in the Inline Pivot layer sigil", - got: previous.value, type: previous.type, line: previous.line) + got: previous.display_value, type: previous.type, line: previous.line) end TypeCapabilities.new( ownership: parsed.ownership || :affine, diff --git a/compiler/ruby/ast/source_error.rb b/compiler/ruby/ast/source_error.rb index c32cd1546..24b4cca57 100644 --- a/compiler/ruby/ast/source_error.rb +++ b/compiler/ruby/ast/source_error.rb @@ -36,9 +36,9 @@ def error!(node_or_token, code_or_message, *args, **kwargs) token = diagnostic_token(node_or_token) # 2. Determine Message - message = T.let("", String) + message = T.let("", T.nilable(String)) if code_or_message.is_a?(Symbol) - message = DiagnosticRegistry.format_from_hash(code_or_message, args, kwargs) || "" + message = DiagnosticRegistry.format_from_hash(code_or_message, args, kwargs) raise "Internal Compiler Error: Unknown error code :#{code_or_message}" unless message else # C. Legacy Support (Raw String) diff --git a/compiler/ruby/ast/std_lib.rb b/compiler/ruby/ast/std_lib.rb index 8f857a68e..6d7b657b4 100644 --- a/compiler/ruby/ast/std_lib.rb +++ b/compiler/ruby/ast/std_lib.rb @@ -24,7 +24,10 @@ "symbol" => { args: [STRING_TYPE], return: {type: STRING_TYPE, sync: :symbol}, - zig: "try {rt}.internSymbol({0})", + # Wrapped, because a Symbol is a distinct type from the []const u8 the + # intern table hands back -- that distinction is what stops cleanup from + # treating an intern-table handle as an owned String. + zig: "CheatLib.symbolOf(try {rt}.internSymbol({0}))", bc: false, allocates: true, needs_rt: true, @@ -617,6 +620,18 @@ is_method: true, }, + # "hELLO world".capitalize() -> "Hello world" + "capitalize" => { + args: [STRING_TYPE], + return: STRING_TYPE, return_alloc: :frame, + zig: "try CheatLib.stringCapitalize({alloc}, {0})", + # No `bc:` — the register VM has no capitalize opcode, and claiming one + # would make MIR lowering emit an InlineBc the emitter cannot compile. + allocates: true, + alloc: :node_storage, + is_method: true, + }, + # contains?("hello", "ll") -> true # contains?(arr, item) -> true/false (linear search, @list or T[]) "contains?" => [ @@ -1623,9 +1638,11 @@ class StdLibTypeBinding < T::Struct eql: { zig: "CheatLib.eql({0}, {1})", bc: true, borrows: :all }, strcmp: { zig: "CheatLib.strcmp({0}, {1})", bc: true, borrows: :all }, strEql: { zig: "CheatLib.strEql({0}, {1})", bc: true, borrows: :all }, - # O(1) pointer+length comparison for String@symbol. Valid for compiler-pooled - # static symbol literals; dynamic String@symbol values must be interned first. - symbolEql: { zig: "({0}.ptr == {1}.ptr and {0}.len == {1}.len)", bc: true, borrows: :all }, + # String@symbol comparison. Symbols have two representations that never share + # a pointer -- compiler-pooled rodata literals and runtime intern-table + # handles from `symbol(runtime_string)` -- so identity alone is wrong. + # std.mem.eql keeps the pointer/length fast path and falls back to content. + symbolEql: { zig: "CheatLib.eql({0}, {1})", bc: true, borrows: :all }, # --- String indexing --- charAt: { zig: "CheatLib.charAt({0}, {1})", bc: true, borrows: :all }, diff --git a/compiler/ruby/ast/symbol_entry.rb b/compiler/ruby/ast/symbol_entry.rb index 44106b50c..9bbf7ae6b 100644 --- a/compiler/ruby/ast/symbol_entry.rb +++ b/compiler/ruby/ast/symbol_entry.rb @@ -50,7 +50,9 @@ class SymbolEntry @next_binding_id = T.let(0, Integer) TypeInput = T.type_alias { T.nilable(T.any(Type::TypeInput, FunctionSignature)) } - RegInput = T.type_alias { T.nilable(T.any(AST::Node, String, Symbol)) } + # A MATCH payload binding records its MatchCase arm as `reg` so lowering can + # rename a nested rebind of the same name; MatchCase is not Locatable. + RegInput = T.type_alias { T.nilable(T.any(AST::Node, AST::MatchCase, String, Symbol)) } LifetimeSourceInput = T.type_alias { T.any(SymbolEntry, Symbol) } LifetimeInput = T.type_alias { T.nilable(T.any(Symbol, T::Array[LifetimeSourceInput], T::Hash[Symbol, T::Array[LifetimeSourceInput]])) } @@ -730,7 +732,8 @@ def self.normalize_type_input(value) sig { params(signature: FunctionSignature).returns(Type) } def self.type_from_function_signature(signature) param_types = signature.params.map(&:type) - Type.function_type_from_parts(param_types, signature.return_type, signature.reentrant, signature) + Type.function_type_from_parts(param_types, signature.return_type, signature.reentrant, signature, + :clear, signature.params.map { |param| param.mutable == true }) end sig { returns(Integer) } diff --git a/compiler/ruby/ast/type.rb b/compiler/ruby/ast/type.rb index 346fe5b7e..d0f2e700b 100644 --- a/compiler/ruby/ast/type.rb +++ b/compiler/ruby/ast/type.rb @@ -67,6 +67,10 @@ class Type # ruby-to-clear: pub class FunctionTypeParam < T::Struct const :type, Type + # A callback can only mutate what its parameter declares mutable, exactly + # like a named function's MUTABLE parameter: the value is passed by pointer + # and the call site marks it with `&`. + const :mutable, T::Boolean, default: false end # ruby-to-clear: pub @@ -86,7 +90,7 @@ class FunctionType < T::Struct sig { params(signature: FunctionType).returns(TypeExpressionKind) } def self.function_type_expression_for(signature) FunctionTypeExpression.new(signature: FunctionSignatureExpression.new( - params: signature.params.map { |param| FunctionParamExpression.new(expression: param.type.shape.expression) }, + params: signature.params.map { |param| FunctionParamExpression.new(expression: param.type.shape.expression, mutable: param.mutable) }, return_expression: signature.return_type.shape.expression, reentrant: signature.reentrant, abi: signature.abi, @@ -102,7 +106,7 @@ def self.function_type_for_expression(expression) return payload unless payload.nil? FunctionType.new( - params: expression.signature.params.map { |param| FunctionTypeParam.new(type: Type.new(param.expression)) }, + params: expression.signature.params.map { |param| FunctionTypeParam.new(type: Type.new(param.expression), mutable: param.mutable) }, return_type: Type.new(expression.signature.return_expression), reentrant: expression.signature.reentrant, abi: expression.signature.abi, @@ -1115,12 +1119,25 @@ def self.stream_step_of(item_type) generic_instance_of(:StreamStep, [item_type]) end - sig { params(param_types: T::Array[Type], return_type: Type, reentrant: T::Boolean, source_signature: T.nilable(BasicObject), abi: Symbol).returns(Type) } - def self.function_type_from_parts(param_types, return_type, reentrant, source_signature, abi = :clear) + sig do + params( + param_types: T::Array[Type], + return_type: Type, + reentrant: T::Boolean, + source_signature: T.nilable(BasicObject), + abi: Symbol, + mutable_flags: T::Array[T::Boolean], + ).returns(Type) + end + def self.function_type_from_parts(param_types, return_type, reentrant, source_signature, abi = :clear, + mutable_flags = []) params = T.let([], T::Array[FunctionTypeParam]) i = T.let(0, Integer) while i < param_types.length - params << FunctionTypeParam.new(type: copy_type(T.must(param_types[i]))) + params << FunctionTypeParam.new( + type: copy_type(T.must(param_types[i])), + mutable: mutable_flags[i] == true, + ) i += 1 end @@ -1493,21 +1510,17 @@ def self.resolve_concat_op(t_left, t_right, left_type, right_type) return BinaryOpResult.new(error: "Operator $+ requires at least one String operand, got #{t_left} and #{t_right}") end - left_coercion = (!left_type.string? && safe_autocast?(t_left, :String)) ? :String : nil - right_coercion = (!right_type.string? && safe_autocast?(t_right, :String)) ? :String : nil - BinaryOpResult.new(type: Type.new(:String), left_coercion: left_coercion, - right_coercion: right_coercion, storage: :frame) - end + # There is no bit-level coercion from a number or a Bool to a string: + # rendering one allocates. Ask for the `.toString()` that does it rather + # than stamping a coercion the emitter can only turn into `@as([]const u8, n)`. + non_string = !left_type.string? ? t_left : (!right_type.string? ? t_right : nil) + if non_string + return BinaryOpResult.new( + error: "Operator $+ requires String operands, got #{non_string} - call .toString() on it", + ) + end - sig { params(from_type: Symbol, to_type: Symbol).returns(T::Boolean) } - def self.safe_autocast?(from_type, to_type) - from_t = Type.new(from_type.to_s.to_sym) - to_t = Type.new(to_type.to_s.to_sym) - return false if from_t.fn_type? || to_t.fn_type? - # Any numeric -> any numeric (implicit promotion/narrowing handled by Zig casts) - return true if from_t.numeric? && to_t.numeric? - # Original types that can auto-cast to strings - [:Float64, :Int64, :Bool, :Byte].include?(from_t.resolved) + BinaryOpResult.new(type: Type.new(:String), storage: :frame) end public @@ -2959,6 +2972,15 @@ def raw? sync == :raw end + # A String-typed slot that renders as bytes ([]const u8) rather than as the + # interned Symbol handle. This is the target every symbol-widening site + # tests for; naming it once keeps the three coercion boundaries (CAST, + # placement, call arguments) from each re-spelling the pair. + sig { returns(T::Boolean) } + def byte_string? + string? && !symbol? + end + sig { returns(T::Boolean) } def symbol? sync == :symbol @@ -3445,9 +3467,11 @@ def soa_list_materialization? (list_collection? || fixed_soa?) && soa? end + # A `T[]` field renders as a Zig slice, but a `[]T@list` field is an + # ArrayList: only the latter iterates through `.items`. sig { returns(T::Boolean) } - def dynamic_field_array? - array? && (dynamic? || list_collection?) + def slice_shaped_field_array? + array? && dynamic? && !list_collection? end sig { returns(T::Boolean) } @@ -4327,15 +4351,18 @@ def implicitly_copyable?(lookup_arg = nil, &lookup_block) # ── Recursive type analysis (mirrors Zig comptime functions) ────── - sig { params(schema_lookup: T.nilable(SchemaLookup), seen: T.nilable(T::Set[String])).returns(T::Boolean) } - def recursive_cleanup_shape?(schema_lookup = nil, seen = nil) + # `ignore_borrow` asks about the POINTEE's shape rather than the borrow's: + # a borrow owns nothing to drop, but COPY through one still has to duplicate + # whatever the pointee owns. + sig { params(schema_lookup: T.nilable(SchemaLookup), seen: T.nilable(T::Set[String]), ignore_borrow: T::Boolean).returns(T::Boolean) } + def recursive_cleanup_shape?(schema_lookup = nil, seen = nil, ignore_borrow: false) return false if node_reference? seen_set = T.let(seen || Set.new, T::Set[String]) key = type_id.key return false if seen_set.include?(key) seen_set << key - return false if borrowed_reference? + return false if borrowed_reference? && !ignore_borrow # Symbols are interned, process-lifetime string data. They have String's # representation, but never own the backing bytes and therefore must not # make an enclosing collection recursively cleanup-bearing. @@ -5280,7 +5307,7 @@ def semantic_shape_key # ruby-to-clear: effects reentrant def function_type_key sig = T.must(function_type) - param_keys = sig.params.map { |param| param.type.semantic_type_key } + param_keys = sig.params.map { |param| param.mutable ? "MUTABLE #{param.type.semantic_type_key}" : param.type.semantic_type_key } params_key = param_keys.join(",") "fn(#{params_key})->#{sig.return_type.semantic_type_key};reentrant=#{sig.reentrant};abi=#{sig.abi}" @@ -5495,6 +5522,10 @@ def map_zig_type return "CheatLib.NumericMapType(#{numeric_key_zig}, #{val_zig})" end + # A symbol value needs no special map: CheatLib.Symbol's drop is a no-op, + # so the ordinary owned-value StringMap leaves intern-table storage alone. + # The old InternedValueStringMap existed because a symbol was spelled + # []const u8 and the map could not tell it from an owned String. "CheatLib.StringMap(#{val_zig})" end @@ -5573,7 +5604,9 @@ def compute_zig_type(is_param: false, is_field: false) while i < fn_raw.params.length p = fn_raw.params.fetch(i) t = p.type - param_types_zig << t.zig_type(is_param: true) + param_zig = t.zig_type(is_param: true) + param_zig = "*#{param_zig}" if p.mutable && !param_zig.start_with?("*") + param_types_zig << param_zig i += 1 end ret_zig = fn_raw.return_type.zig_type @@ -5607,6 +5640,11 @@ def compute_zig_type(is_param: false, is_field: false) return signed_integer? ? "isize" : "usize" end if resolved == :String || string? + # An interned symbol is represented exactly like a String and owned by + # nobody. Spelling both []const u8 left every downstream consumer -- + # cleanup above all -- unable to tell them apart. + return "CheatLib.Symbol" if symbol? + return "[]const u8" end @@ -5639,10 +5677,8 @@ def compute_zig_type(is_param: false, is_field: false) # 3d. Handle @set collection if set_collection? elem = T.must(element_type) - # Interned symbols are rodata/intern-table handles the set never - # owns; the owned-string Set would free them on duplicate insert - # and at deinit (misaligned free of rodata). - return "CheatLib.InternedStringSet()" if elem.symbol? + # Same for a set of symbols: Symbol drops to nothing, so the ordinary + # Set is correct without a separate interned representation. base_zig = elem.nested_zig_type(is_param: is_param, is_field: is_field) return "CheatLib.Set(#{base_zig})" end @@ -5806,7 +5842,9 @@ def self.from_function_signature(signature) param_types, return_type.is_a?(Type) ? return_type : Type.new(return_type), raw.reentrant, - signature + signature, + :clear, + raw.params.map { |param| param.mutable == true }, ) end end @@ -5994,22 +6032,16 @@ class ResourceSchema sig { returns(Schemas::ResourceClosePlan) } attr_reader :close_plan - sig { returns(Schemas::ResourceSchema::StaticMethodsMap) } attr_reader :static_methods - sig { returns(T::Hash[String, AST::StructField]) } attr_reader :fields - sig { returns(T.nilable(String)) } attr_reader :extern_module - sig { returns(T.nilable(String)) } attr_reader :as_type - sig { returns(Symbol) } attr_reader :visibility - sig { returns(Schemas::ResourceSchema::MethodsMap) } attr_reader :methods sig { returns(T::Array[Symbol]) } @@ -6187,7 +6219,6 @@ class UnionSchema sig { returns(Schemas::UnionSchema::VariantMap) } attr_reader :variants - sig { returns(Symbol) } attr_reader :visibility sig { returns(T::Array[Symbol]) } @@ -6256,16 +6287,12 @@ class StructSchema sig { returns(T::Hash[String, AST::StructField]) } attr_reader :fields - sig { returns(MethodsMap) } attr_reader :methods - sig { returns(Symbol) } attr_reader :visibility - sig { returns(T.nilable(String)) } attr_reader :extern_module - sig { returns(T.nilable(String)) } attr_reader :as_type sig { returns(T::Array[Symbol]) } diff --git a/compiler/ruby/ast/type_expression.rb b/compiler/ruby/ast/type_expression.rb index 87f53a647..5d5188aee 100644 --- a/compiler/ruby/ast/type_expression.rb +++ b/compiler/ruby/ast/type_expression.rb @@ -66,6 +66,8 @@ class TypeProjectionExpression < T::Struct class FunctionParamExpression < T::Struct const :expression, TypeExpression + # `FN(MUTABLE T) -> R`: the callback may mutate this parameter. + const :mutable, T::Boolean, default: false end # Foundation-native function signature: parameters and return spelled as @@ -362,7 +364,7 @@ def self.transform(expression, &visitor) TypeExpression.new(kind: FunctionTypeExpression.new( signature: FunctionSignatureExpression.new( params: signature.params.map do |param| - FunctionParamExpression.new(expression: transform(param.expression, &visitor)) + FunctionParamExpression.new(expression: transform(param.expression, &visitor), mutable: param.mutable) end, return_expression: transform(signature.return_expression, &visitor), reentrant: signature.reentrant, diff --git a/compiler/ruby/backends/mir_emitter.rb b/compiler/ruby/backends/mir_emitter.rb index d49de5136..ec16ac08a 100644 --- a/compiler/ruby/backends/mir_emitter.rb +++ b/compiler/ruby/backends/mir_emitter.rb @@ -547,7 +547,7 @@ def emit_context_field_decls(fields) sig { params(fields: T::Array[MIR::StructInitField]).returns(String) } def emit_struct_init_fields(fields) fields.map do |field| - ".#{field.name} = #{emit(field.value)}" + ".#{zig_field_name(field.name)} = #{emit_struct_init_field_value(field.value)}" end.join(", ") end @@ -1296,7 +1296,7 @@ def emit_polymorphic_mutate(node) cell_zig = T.must(emit(node.cell)) capture_param = captures.empty? ? "" : ", __captures: anytype" capture_suppress = captures.empty? ? "" : "_ = &__captures;" - all_capture_args = captures + guard_captures.map { |name| "&#{name}_moved" } + all_capture_args = captures + guard_captures.map { |name| "&#{move_guard_name(name)}" } capture_args = all_capture_args.empty? ? ".{}" : ".{.{#{all_capture_args.join(', ')}}}" <<~ZIG.rstrip try CheatLib.polymorphicMutate(#{cell_zig}, #{node.rt}, struct { @@ -1332,7 +1332,7 @@ def emit_polymorphic_mutate_flow(node) end capture_param = captures.empty? ? "" : ", __captures: anytype" capture_suppress = captures.empty? ? "" : "_ = &__captures;" - all_capture_args = captures + guard_captures.map { |name| "&#{name}_moved" } + all_capture_args = captures + guard_captures.map { |name| "&#{move_guard_name(name)}" } capture_args = all_capture_args.empty? ? ".{&__poly_flow}" : ".{&__poly_flow, .{#{all_capture_args.join(', ')}}}" guard_block = "" if node.guard_cond @@ -2010,7 +2010,9 @@ def symbol_pool_declarations end lines << "// Static String@symbol literal pool." unless @symbol_literals.empty? @symbol_literals.each do |value, name| - lines << "const #{name}: []const u8 = #{zig_byte_string_literal(value)};" + # A symbol constant is a Symbol, not a slice: that is what keeps cleanup + # from mistaking the .rodata behind it for an owned String. + lines << "const #{name}: CheatLib.Symbol = .{ .bytes = #{zig_byte_string_literal(value)} };" end lines.join("\n") end @@ -2047,7 +2049,7 @@ def emit_struct_def(node) vis = node.visibility == :pub ? "pub " : "" fields = (node.fields || []).map { |f| default = f.default ? " = #{emit(f.default)}" : "" - "#{f.name}: #{f.zig_type}#{default}," + "#{zig_field_name(f.name)}: #{f.zig_type}#{default}," }.join("\n ") methods = (node.methods || []).map { |m| emit(m) }.join("\n\n ") @@ -2219,7 +2221,16 @@ def emit_set(node) sig { params(node: MIR::DestructureSet).returns(String) } def emit_destructure_set(node) targets = node.targets.map { |target| T.must(emit(target)) }.join(", ") - "#{targets} = #{emit(node.value)};" + # A `var` binding Zig never sees mutated is an error, and a destructure + # target is bound in one statement with no later `_ = &name;` to vouch for + # it. Ordinary Let emission already appends the same suppression. + suppressions = node.targets.filter_map do |target| + next unless target.is_a?(MIR::DestructureTarget) && target.declaration_kind == :var + next if target.name.to_s == "_" + + " _ = &#{target.name};" + end.join + "#{targets} = #{emit(node.value)};#{suppressions}" end sig { params(node: MIR::DestructureTarget).returns(String) } @@ -2235,11 +2246,49 @@ def emit_destructure_target(node) end end + # A binding name may already be an escaped Zig identifier (`@"f2"` for a name + # Zig would read as a primitive type). Every identifier DERIVED from one -- + # a temp, a move guard -- must be built from the bare base, or the escape + # lands in the middle of the new name and does not parse. + sig { params(name: T.any(String, Symbol)).returns(String) } + def zig_bare_name(name) + text = name.to_s + text.start_with?('@"') && text.end_with?('"') ? T.must(text[2..-2]) : text + end + + sig { params(name: T.any(String, Symbol)).returns(String) } + def move_guard_name(name) + "#{zig_bare_name(name)}_moved" + end + + # A struct field named after a Zig keyword needs the escaped spelling in the + # declaration and at every access: `comptime: bool` parses as a comptime + # field, and `node.comptime` as the start of a comptime block. + # A struct-literal field has a known type, so `null` needs no `@as`. Emitting + # one names the field's type -- which may live in a package this module never + # imported, and then does not resolve. The value is what matters here, not a + # redundant annotation Zig infers anyway. + sig { params(value: T.untyped).returns(String) } + def emit_struct_init_field_value(value) + inner = value.is_a?(MIR::Cast) && value.method == :as ? value.expr : nil + return "null" if inner.is_a?(MIR::Lit) && inner.value.to_s == "null" + + T.must(emit(value)) + end + + sig { params(name: T.any(String, Symbol)).returns(String) } + def zig_field_name(name) + text = name.to_s + return text if text.start_with?('@"') + ZigType.reserved_identifier?(text) ? "@\"#{text}\"" : text + end + sig { params(node: MIR::ReassignWithCleanup).returns(String) } def emit_reassign_cleanup(node) + base = zig_bare_name(node.name) if (try_expr = reassign_success_only_expr(node)) - opt = "__new_#{node.name}_opt" - val = "__new_#{node.name}_val" + opt = "__new_#{base}_opt" + val = "__new_#{base}_val" alloc = alloc_zig(node.alloc) return [ "{", @@ -2252,7 +2301,7 @@ def emit_reassign_cleanup(node) ].join("\n") end - tmp = "__new_#{node.name}" + tmp = "__new_#{base}" val = emit(node.value) alloc = alloc_zig(node.alloc) "{\nconst #{tmp} = #{val};\nCheatLib.cleanup(@TypeOf(#{node.name}), #{alloc}, &#{node.name});\n#{node.name} = #{tmp};\n}" @@ -2865,7 +2914,7 @@ def emit_direct_cleanup(name, entry, alloc_override: nil) use_type = via_pointer ? "@TypeOf(#{use_name}.*)" : "@TypeOf(#{use_name})" result = direct_uniform_cleanup(use_name, use_type, use_alloc, guarded, via_pointer:) if entry.rc_release_fields_cleanup? - guard = guarded ? "if (!#{name}_moved) " : "" + guard = guarded ? "if (!#{move_guard_name(name)}) " : "" result += "\n#{guard}CheatLib.releaseFields(#{entry.base_zig}, #{use_alloc}, #{name}.ctrl.data.*);" end result @@ -2916,7 +2965,7 @@ def emit_cleanup(node, errdefer: false) use_type = vp ? "@TypeOf(#{use_name}.*)" : "@TypeOf(#{use_name})" result = guarded_cleanup(use_name, use_type, use_alloc, g, errdefer:, via_pointer: vp) if entry.rc_release_fields_cleanup? - guard = g ? "if (!#{name}_moved) " : "" + guard = g ? "if (!#{move_guard_name(name)}) " : "" kw = errdefer ? "errdefer" : "defer" result += "#{kw} #{guard}CheatLib.releaseFields(#{entry.base_zig}, #{use_alloc}, #{name}.ctrl.data.*);\n" end @@ -2926,13 +2975,17 @@ def emit_cleanup(node, errdefer: false) sig { params(node: MIR::MoveMark).returns(String) } def emit_move_mark(node) - guard = @move_guard_overrides.fetch(node.name.to_s, "#{node.name}_moved") + guard = @move_guard_overrides.fetch(node.name.to_s, move_guard_name(node.name)) "#{guard} = true;" end sig { params(node: MIR::DeepCopy).returns(T.nilable(String)) } def emit_deep_copy(node) - src = emit(node.source) + src = T.must(emit(node.source)) + # A noreturn source has nothing to duplicate: binding it to a copy temp + # emits `const __copy_src = @panic(...)`, which is unreachable code. + return src if src.start_with?("@panic(") + alloc = node.alloc ? alloc_expr(node.alloc) : nil # Uniquify the blk label across nested DeepCopy emits in the same scope. @deep_copy_counter += 1 @@ -3133,7 +3186,7 @@ def emit_call_argument(argument) def emit_field_get(node) object = T.must(emit(node.object)) object = "(#{object})" if node.object.is_a?(MIR::StructInit) || node.object.is_a?(MIR::TupleLiteral) - "#{paren_if_try(object)}.#{node.field}" + "#{paren_if_try(object)}.#{zig_field_name(node.field)}" end sig { params(node: MIR::UnionPayloadGet).returns(String) } @@ -3264,7 +3317,7 @@ def emit_struct_init(node) value = MIR.struct_init_field_value(field) next nil unless name && value - ".#{name} = #{emit(value)}" + ".#{zig_field_name(name)} = #{emit_struct_init_field_value(value)}" end.join(", ") if node.zig_type "#{node.zig_type}{ #{fields} }" @@ -3333,7 +3386,7 @@ def emit_concat(node) sig { params(node: MIR::Cast).returns(String) } def emit_cast(node) - inner = emit(node.expr) + inner = T.must(emit(node.expr)) # `@as(!T, ...)` and `@as(!?T, ...)` parse as `@as(boolean_not, ...)` # in expression context. Force type interpretation by prefixing with # `anyerror`. (Same workaround as Promise(anyerror!T) in type.rb's @@ -3341,6 +3394,10 @@ def emit_cast(node) # call site.) target_t = node.target_type target_t = ZigType.new(target_t).cast_target_type if target_t + # `@as(T, @panic("..."))` is unreachable code: a noreturn value already + # coerces to every type, so the annotation only breaks the build. + return inner if inner.start_with?("@panic(") + case node.method when :as "@as(#{target_t}, #{inner})" @@ -3368,9 +3425,11 @@ def emit_cast(node) sig { params(node: MIR::Orelse).returns(String) } def emit_orelse(node) - fallback = emit(node.fallback) + fallback = T.must(emit(node.fallback)) result_type = node.result_type - fallback = "@as(#{result_type.zig_type}, #{fallback})" if result_type + # A noreturn fallback (`OR_ELSE panic("...")`) already coerces to the + # result type; annotating it makes the whole expression unreachable code. + fallback = "@as(#{result_type.zig_type}, #{fallback})" if result_type && !fallback.start_with?("@panic(") expr = emit(node.expr) expr = "@as(?#{result_type.zig_type}, null)" if result_type && expr == "null" "(#{expr} orelse #{fallback})" @@ -3605,7 +3664,7 @@ def unique_heap_allocator_cache_name def guarded_defer(name, body, guarded, errdefer: false) kw = errdefer ? "errdefer" : "defer" if guarded - "var #{name}_moved = false; _ = &#{name}_moved;\n#{kw} if (!#{name}_moved) #{body};\n" + "var #{move_guard_name(name)} = false; _ = &#{move_guard_name(name)};\n#{kw} if (!#{move_guard_name(name)}) #{body};\n" elsif body.start_with?("{") && body.end_with?("}") "#{kw} #{body}\n" else @@ -3680,7 +3739,7 @@ def emit_resource_close(node) def direct_cleanup_statement(name, body, guarded) stripped = body.strip statement = stripped.end_with?(";", "}") ? stripped : "#{stripped};" - guarded ? "if (!#{name}_moved) #{statement}" : statement + guarded ? "if (!#{move_guard_name(name)}) #{statement}" : statement end sig { params(name: String, zig_type: String, alloc: String, guarded: T::Boolean, via_pointer: T.nilable(T::Boolean)).returns(String) } diff --git a/compiler/ruby/backends/transpiler.rb b/compiler/ruby/backends/transpiler.rb index c1333e363..8f8c9541c 100644 --- a/compiler/ruby/backends/transpiler.rb +++ b/compiler/ruby/backends/transpiler.rb @@ -314,6 +314,12 @@ def transpile_as_module(cheat_code, source_dir: @source_dir, pkg_paths: {}) "" end + # A module that RAISEs names `ErrorName.`, so it needs the same + # per-program enum the root emits. Ids come from the shared registry: the + # stdlib seed is fixed and user types are numbered in first-use order over + # the module's import closure, which every module in a package shares. + error_name_enum = body.include?("ErrorName.") ? "#{emit_error_name_enum}\n" : "" + <<~ZIG const std = @import("std"); const CheatHeader = @import("cheat_runtime"); @@ -321,6 +327,7 @@ def transpile_as_module(cheat_code, source_dir: @source_dir, pkg_paths: {}) const Runtime = CheatHeader.Runtime; const EbrContext = CheatHeader.EbrContext; #{safety_line} + #{error_name_enum} #{body} #{test_block} ZIG diff --git a/compiler/ruby/compiler/module_importer.rb b/compiler/ruby/compiler/module_importer.rb index 6c44ec35f..143278f64 100644 --- a/compiler/ruby/compiler/module_importer.rb +++ b/compiler/ruby/compiler/module_importer.rb @@ -3,6 +3,7 @@ require "set" require_relative "package_source" +require_relative "../incremental/module_cache" class ModuleImportError < StandardError; end class CircularDependencyError < ModuleImportError; end @@ -60,6 +61,8 @@ def initialize(base_dir: Dir.pwd, pkg_paths: {}, use_mir: false, stdlib_root: ST # member file (directly or via its own single-file pkg name) is aliased # to the whole package so the unit is never split. @package_members = T.let({}, T::Hash[String, String]) + # Cross-run store for compiled units. Nil unless `clear` asked for one. + @unit_cache = T.let(Incremental::ModuleCache.from_env, T.nilable(Incremental::ModuleCache)) @pkg_paths.each do |name, value| next unless value.to_s.include?(",") @@ -75,6 +78,17 @@ def initialize(base_dir: Dir.pwd, pkg_paths: {}, use_mir: false, stdlib_root: ST # 2. First-party stdlib at //src/lib.clear # # @param pkg_name [String] Package name (e.g. "math", "testing") + # The package that actually owns a required name. A single-file package whose + # file belongs to a multi-file package IS that package -- and only the owner + # is built, so the emitted Zig must import (and alias through) the owner. + sig { params(pkg_name: String).returns(String) } + def owning_package_name(pkg_name) + path = @pkg_paths[pkg_name.to_s] + return pkg_name.to_s if path.nil? || path.to_s.include?(",") + + @package_members[File.expand_path(path.to_s)] || pkg_name.to_s + end + sig { params(pkg_name: String, caller_dir: String).returns(T.nilable(ModuleImporter::CompiledModule)) } def compile_package(pkg_name, caller_dir: @base_dir) path = @pkg_paths[pkg_name.to_s] || resolve_stdlib_package(pkg_name) @@ -100,7 +114,10 @@ def compile_package(pkg_name, caller_dir: @base_dir) sig { params(pkg_name: String, members: T::Array[String]).returns(T.nilable(ModuleImporter::CompiledModule)) } def compile_package_group(pkg_name, members) cache_key = "pkg-group:#{pkg_name}" - return @module_cache[cache_key] if @module_cache.key?(cache_key) + if @module_cache.key?(cache_key) + @unit_cache&.reuse(cache_key) + return @module_cache[cache_key] + end if @compiling.include?(cache_key) cycle = @compiling.to_a.map { |p| File.basename(p.to_s) }.join(" -> ") @@ -114,26 +131,28 @@ def compile_package_group(pkg_name, members) @compiling.add(cache_key) begin - merged = PackageSource.merge(members, resolve_pkg: ->(name) { @pkg_paths[name] || resolve_stdlib_package(name) }) - source_dir = File.dirname(T.must(merged.member_paths.first)) - - saved_gradual = ClearParser.gradual_mode - ClearParser.gradual_mode = false - ast = begin - budget = FrontendResourceBudget.new - tokens = Lexer.new(merged.source, file: "pkg:#{pkg_name}", budget: budget).tokenize - ClearParser.new(tokens, merged.source, budget: budget).parse - ensure - ClearParser.gradual_mode = saved_gradual + mod = with_unit_cache(cache_key, members) do + merged = PackageSource.merge(members, resolve_pkg: ->(name) { @pkg_paths[name] || resolve_stdlib_package(name) }) + source_dir = File.dirname(T.must(merged.member_paths.first)) + + saved_gradual = ClearParser.gradual_mode + ClearParser.gradual_mode = false + ast = begin + budget = FrontendResourceBudget.new + tokens = Lexer.new(merged.source, file: "pkg:#{pkg_name}", budget: budget).tokenize + ClearParser.new(tokens, merged.source, budget: budget).parse + ensure + ClearParser.gradual_mode = saved_gradual + end + + reject_auto_in_public_signatures!(ast, "pkg:#{pkg_name}") + + annotator = SemanticAnnotator.new(importer: self, source_dir: source_dir, source_code: merged.source) + annotator.annotate!(ast) + + compile_module_mir(ast, annotator, source_dir) end - reject_auto_in_public_signatures!(ast, "pkg:#{pkg_name}") - - annotator = SemanticAnnotator.new(importer: self, source_dir: source_dir, source_code: merged.source) - annotator.annotate!(ast) - - mod = compile_module_mir(ast, annotator, source_dir) - @module_cache[cache_key] = mod mod ensure @@ -174,7 +193,10 @@ def compile_file(path, caller_dir: @base_dir) owner = @package_members[abs_path] return compile_package(owner, caller_dir: caller_dir) if owner - return @module_cache[abs_path] if @module_cache.key?(abs_path) + if @module_cache.key?(abs_path) + @unit_cache&.reuse(abs_path) + return @module_cache[abs_path] + end if @compiling.include?(abs_path) cycle = @compiling.to_a.map { |p| File.basename(p) }.join(" -> ") @@ -185,32 +207,34 @@ def compile_file(path, caller_dir: @base_dir) @compiling.add(abs_path) begin - source = File.read(abs_path) - source_dir = File.dirname(abs_path) - - # STRICT-imports boundary (gradual-typing.md §7): imported modules - # must export concrete types in their public surface. Force the - # parser into strict mode (gradual=false) for the duration of the - # imported module's parse so `--gradual` from the top-level build - # never propagates across module boundaries. Explicit `Auto` in - # source still tokenizes; the post-parse check below catches it. - saved_gradual = ClearParser.gradual_mode - ClearParser.gradual_mode = false - ast = begin - budget = FrontendResourceBudget.new - tokens = Lexer.new(source, file: abs_path, budget: budget).tokenize - ClearParser.new(tokens, source, budget: budget).parse - ensure - ClearParser.gradual_mode = saved_gradual + mod = with_unit_cache(abs_path, [abs_path]) do + source = File.read(abs_path) + source_dir = File.dirname(abs_path) + + # STRICT-imports boundary (gradual-typing.md §7): imported modules + # must export concrete types in their public surface. Force the + # parser into strict mode (gradual=false) for the duration of the + # imported module's parse so `--gradual` from the top-level build + # never propagates across module boundaries. Explicit `Auto` in + # source still tokenizes; the post-parse check below catches it. + saved_gradual = ClearParser.gradual_mode + ClearParser.gradual_mode = false + ast = begin + budget = FrontendResourceBudget.new + tokens = Lexer.new(source, file: abs_path, budget: budget).tokenize + ClearParser.new(tokens, source, budget: budget).parse + ensure + ClearParser.gradual_mode = saved_gradual + end + + reject_auto_in_public_signatures!(ast, abs_path) + + annotator = SemanticAnnotator.new(importer: self, source_dir: source_dir, source_code: source) + annotator.annotate!(ast) + + compile_module_mir(ast, annotator, source_dir) end - reject_auto_in_public_signatures!(ast, abs_path) - - annotator = SemanticAnnotator.new(importer: self, source_dir: source_dir, source_code: source) - annotator.annotate!(ast) - - mod = compile_module_mir(ast, annotator, source_dir) - @module_cache[abs_path] = mod mod ensure @@ -258,6 +282,20 @@ def auto_type?(t) private + # Route one compilation unit through the cross-run unit cache when one is + # configured. Every REQUIRE the block issues re-enters the importer, so the + # cache sees the unit's transitive source set without a second dependency scan. + sig do + params(unit_key: String, member_paths: T::Array[String], block: T.proc.returns(ModuleImporter::CompiledModule)) + .returns(ModuleImporter::CompiledModule) + end + def with_unit_cache(unit_key, member_paths, &block) + cache = @unit_cache + return block.call unless cache + + cache.fetch(unit_key, member_paths, &block) + end + sig { params(ast: AST::Program, annotator: SemanticAnnotator, source_dir: String).returns(ModuleImporter::CompiledModule) } def compile_module_mir(ast, annotator, source_dir) fn_nodes = prepare_module_mir!(ast, annotator) diff --git a/compiler/ruby/incremental/module_cache.rb b/compiler/ruby/incremental/module_cache.rb new file mode 100644 index 000000000..8f2620226 --- /dev/null +++ b/compiler/ruby/incremental/module_cache.rb @@ -0,0 +1,191 @@ +# typed: strict +# frozen_string_literal: true + +require "digest" +require "fileutils" +require "sorbet-runtime" + +module Incremental + # On-disk cache of compiled REQUIRE units, keyed by content. + # + # `clear`'s existing transpile cache keys the WHOLE program on the union of + # its sources, so touching one file recompiles every imported module. This + # cache sits one level down: each unit (a file, or a multi-file package + # group) is stored under a key derived from its own sources, and a stored + # record stays valid while every source it transitively read is unchanged. + # Editing one module then recompiles that module and its dependents only. + # + # A stored unit is a Marshal image of ModuleImporter::CompiledModule. That + # graph is plain compiler data apart from intrinsic `validate:` lambdas, + # which FunctionSignature::AnalysisFacts serializes by registry name. + class ModuleCache + extend T::Sig + + DIR_ENV = "CLEAR_MODULE_CACHE_DIR" + KEY_ENV = "CLEAR_MODULE_CACHE_KEY" + FORMAT_VERSION = "1" + # A unit image is large (whole annotated AST) and every compiler edit + # starts a fresh generation, so cap the directory and drop the coldest + # records rather than filling the disk. One self-hosted-parser generation + # is ~330MB, so this holds a few and no more. + MAX_BYTES = T.let(512 * 1024 * 1024, Integer) + + SourceDigests = T.type_alias { T::Hash[String, String] } + + # Configured cache, or nil when the environment does not ask for one. + # Only `clear` sets these: every other entry point (specs, fmt, fix) + # keeps compiling from scratch. + sig { returns(T.nilable(ModuleCache)) } + def self.from_env + dir = ENV[DIR_ENV] + key = ENV[KEY_ENV] + return nil if dir.nil? || dir.empty? || key.nil? || key.empty? + + new(dir: dir, compiler_key: key) + end + + sig { params(dir: String, compiler_key: String).void } + def initialize(dir:, compiler_key:) + @dir = T.let(File.expand_path(dir), String) + @compiler_key = T.let(compiler_key, String) + # Sources read while compiling the unit currently on top of the stack, + # so a stored record knows its whole transitive input set. + @frames = T.let([], T::Array[SourceDigests]) + @digests = T.let({}, SourceDigests) + # What each unit read, so a unit the importer serves from its in-process + # cache still contributes its sources to whoever imports it next. + @sources_by_unit = T.let({}, T::Hash[String, SourceDigests]) + end + + # Record a unit the importer resolved without calling `fetch` -- its + # in-process cache already had it. Skipping this would let the enclosing + # unit be stored with an incomplete source list, and so be reused after one + # of those sources changed. + sig { params(unit_key: String).void } + def reuse(unit_key) + sources = @sources_by_unit[unit_key] + record_sources(sources) if sources + nil + end + + # Return the stored unit when every source behind it is unchanged, + # otherwise compile it and store the result. + sig do + type_parameters(:U) + .params(unit_key: String, member_paths: T::Array[String], block: T.proc.returns(T.type_parameter(:U))) + .returns(T.type_parameter(:U)) + end + def fetch(unit_key, member_paths, &block) + own = T.let({}, SourceDigests) + member_paths.each { |path| own[File.expand_path(path)] = digest_of(File.expand_path(path)) } + path = record_path(unit_key, own) + + stored = load_record(path) + if stored + sources = T.cast(stored.fetch("sources"), SourceDigests) + @sources_by_unit[unit_key] = sources + record_sources(sources) + return T.unsafe(stored.fetch("unit")) + end + + @frames.push({}) + unit = begin + block.call + rescue StandardError + # A failed compile still leaves the stack balanced; nothing is stored. + @frames.pop + raise + end + sources = T.must(@frames.pop).merge(own) + @sources_by_unit[unit_key] = sources + store_record(path, sources, unit) + record_sources(sources) + unit + end + + private + + # Fold a finished unit's sources into whatever unit is compiling it. + sig { params(sources: SourceDigests).void } + def record_sources(sources) + parent = @frames.last + parent&.merge!(sources) + nil + end + + sig { params(path: String).returns(String) } + def digest_of(path) + cached = @digests[path] + return cached if cached + + @digests[path] = File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing" + end + + sig { params(unit_key: String, own: SourceDigests).returns(String) } + def record_path(unit_key, own) + digest = Digest::SHA256.hexdigest( + [FORMAT_VERSION, @compiler_key, unit_key, own.sort.flatten.join("\0")].join("\0") + ) + File.join(@dir, "#{digest}.unit") + end + + sig { params(path: String).returns(T.nilable(T::Hash[String, T.untyped])) } + def load_record(path) + return nil unless File.file?(path) + + record = T.let(Marshal.load(File.binread(path)), T.untyped) + return nil unless record.is_a?(Hash) + + sources = record["sources"] + return nil unless sources.is_a?(Hash) + # A record is only usable while every source it read still hashes the + # same, which is what makes a dependency edit invalidate its dependents. + return nil unless sources.all? { |source, digest| digest_of(source) == digest } + + record + rescue ArgumentError, TypeError, Errno::ENOENT + # Stale image from an older compiler build: recompile and overwrite. + nil + end + + sig { params(path: String, sources: SourceDigests, unit: T.untyped).void } + def store_record(path, sources, unit) + bytes = Marshal.dump({ "sources" => sources, "unit" => unit }) + FileUtils.mkdir_p(@dir) + temporary = "#{path}.tmp.#{Process.pid}" + begin + File.binwrite(temporary, bytes) + File.rename(temporary, path) + ensure + FileUtils.rm_f(temporary) + end + prune! + rescue TypeError => error + # Something in the graph is not serializable. Compilation is still + # correct without a stored record, so warn once and carry on. + warn "[clear] module cache disabled for #{File.basename(path)}: #{error.message}" + end + + # Drop the least recently used records once the directory outgrows its cap. + sig { void } + def prune! + records = Dir.glob(File.join(@dir, "*.unit")).filter_map do |path| + stat = File.stat(path) + [path, stat.size, stat.mtime] + rescue Errno::ENOENT + nil + end + total = records.sum { |record| record[1] } + return if total <= MAX_BYTES + + records.sort_by! { |record| record[2] } + records.each do |path, size, _mtime| + break if total <= MAX_BYTES + + FileUtils.rm_f(path) + total -= size + end + nil + end + end +end diff --git a/compiler/ruby/incremental/zig_compiler.rb b/compiler/ruby/incremental/zig_compiler.rb index 30473c09f..ed60a20dc 100644 --- a/compiler/ruby/incremental/zig_compiler.rb +++ b/compiler/ruby/incremental/zig_compiler.rb @@ -18,6 +18,7 @@ class ZigCompilerConfig < T::Struct const :test_mode, T::Boolean, default: false const :strict_test, T::Boolean, default: false const :default_stack, T.nilable(String), default: nil + const :main_tier, T.nilable(Symbol), default: nil const :ownership_mode, Symbol, default: :default end @@ -66,7 +67,7 @@ def compile(source, function_counter_seeds: {}) test_mode: @config.test_mode, strict_test: @config.strict_test, exact_tiers: {}, - main_tier: nil, + main_tier: @config.main_tier, default_stack: @config.default_stack, ownership_mode: @config.ownership_mode, function_counter_seeds: function_counter_seeds, diff --git a/compiler/ruby/mir/hoist.rb b/compiler/ruby/mir/hoist.rb index 64983a628..4c13c092c 100644 --- a/compiler/ruby/mir/hoist.rb +++ b/compiler/ruby/mir/hoist.rb @@ -136,7 +136,9 @@ def self.collect_stmt_hoists!(stmt, hoists, counter, schema_lookup, return_type: call.args.each_with_index do |arg, idx| next if arg.is_a?(AST::MoveNode) && arg.value.is_a?(AST::Identifier) next unless allocating?(arg, schema_lookup) - replacement = make_temp!(arg, hoists, counter.next_name, moved: moved_arg?(arg), schema_lookup: schema_lookup) + expected = empty_list_literal?(arg) ? callee_param_type(call, idx) : nil + replacement = make_temp!(arg, hoists, counter.next_name, moved: moved_arg?(arg), + expected_type: expected, schema_lookup: schema_lookup) if call.is_a?(AST::FuncCall) call.args[idx] = replacement elsif call.is_a?(AST::MethodCall) @@ -164,7 +166,9 @@ def self.collect_stmt_hoists!(stmt, hoists, counter, schema_lookup, return_type: expected = Type.from_node(return_type) expected = expected.success_type if expected value_type = Type.from_node!(stmt.value, context: "return value hoist") - expected = value_type if value_type&.collection? + # An empty literal's own collection type is a guess (List) -- the + # declared return type is the only real word on its element. + expected = value_type if value_type&.collection? && !empty_list_literal?(stmt.value) if stmt.value.is_a?(AST::BinaryOp) && (stmt.value.op == :OR || stmt.value.op == :OR_ELSE) right_type = stmt.value.right.is_a?(AST::Locatable) ? stmt.value.right.full_type! : Type.from_node!(stmt.value.right, context: "return OR right hoist") expected = right_type if right_type&.collection? @@ -190,6 +194,9 @@ def self.collect_stmt_hoists!(stmt, hoists, counter, schema_lookup, return_type: sig { params(value: AST::Node, hoists: T::Array[AST::VarDecl], counter: HoistCounter, schema_lookup: T.nilable(Proc), expected_type: T.nilable(Type::TypeInput)).returns(AST::Node) } def self.hoist_escape_value!(value, hoists, counter, schema_lookup, expected_type: nil) return T.cast(value, AST::Node) if value.is_a?(AST::MoveNode) && value.value.is_a?(AST::Identifier) + # A NoReturn expression (`RETURN panic("...")`) escapes nothing -- binding + # it emits `const t = @panic(...)`, which Zig rejects as unreachable code. + return T.cast(value, AST::Node) if noreturn_value?(value) if allocating?(value, schema_lookup) return make_temp!(value, hoists, counter.next_name, expected_type: expected_type) end @@ -279,8 +286,26 @@ def self.each_call_like_child(child, matches, &blk) # For a body-bearing control-flow node, the expression members that # are NOT statement bodies. Plain nodes recurse through their fields normally. + # A pipeline stage's element expression is a per-iteration body lowered in + # its own loop scope with `_` bound. Hoisting an allocating sub-expression + # out of it moves the work out of the loop AND strands the placeholder, + # which the emitted Zig then reads as an undeclared `@"_"`. + PIPELINE_STAGE_NODES = T.let([ + AST::SelectOp, AST::WhereOp, AST::EachOp, AST::TapOp, AST::AllOp, AST::AnyOp, + AST::FindOp, AST::CountOp, AST::SumOp, AST::AverageOp, AST::MinOp, AST::MaxOp, + AST::TakeWhileOp, AST::SkipOp, AST::OrderByOp, AST::DistinctOp, AST::UnnestOp, + AST::IndexOp, AST::ReduceOp, + ].freeze, T::Array[T.untyped]) + + sig { params(node: T.nilable(AST::Node)).returns(T::Boolean) } + def self.pipeline_stage_node?(node) + PIPELINE_STAGE_NODES.any? { |klass| node.is_a?(klass) } + end + sig { params(node: AST::Node).returns(T::Array[BasicObject]) } def self.non_body_exprs(node) + return [] if pipeline_stage_node?(node) + case node when AST::IfStatement, AST::WhileLoop, AST::WhileBindLoop [node.condition] @@ -331,6 +356,21 @@ def self.composite_element_store?(call) !!(et && !et.primitive? && !et.string?) end + sig { params(node: T.nilable(AST::Node)).returns(T::Boolean) } + def self.empty_list_literal?(node) + node.is_a?(AST::ListLit) && node.items.empty? + end + + # An argument hoisted into its own binding loses the call site that gave it a + # type. An empty literal has nothing else to go on, so carry the parameter's + # type onto the temp. + sig { params(call: AST::Node, idx: Integer).returns(T.nilable(Type)) } + def self.callee_param_type(call, idx) + return nil unless call.respond_to?(:matched_signature) + signature = FunctionSignature.unwrap(call.matched_signature) + signature&.params&.[](idx)&.type + end + sig { params(call: AST::MethodCall).returns(T::Boolean) } def self.collection_value_store_call?(call) sig = FunctionSignature.unwrap(call.matched_stdlib_def) @@ -342,6 +382,14 @@ def self.collection_value_store_call?(call) !!(ti.is_a?(Type) && ti.collection?) end + sig { params(node: T.nilable(AST::Node)).returns(T::Boolean) } + def self.noreturn_value?(node) + return false unless node.respond_to?(:resolved_type) + resolved = T.unsafe(node).resolved_type + resolved = resolved.resolved if resolved.is_a?(Type) + resolved == :NoReturn + end + sig { params(node: T.nilable(AST::Node)).returns(T::Boolean) } def self.concat?(node) node.is_a?(AST::StringConcat) || @@ -446,10 +494,12 @@ def self.ast_access_path?(ast_node) :hoist_escape_value! private_class_method :allocating? private_class_method :ast_access_path? + private_class_method :callee_param_type private_class_method :ast_container_borrow_expr? private_class_method :collection_value_store_call? private_class_method :composite_element_store? private_class_method :concat? + private_class_method :empty_list_literal? private_class_method :each_call private_class_method :each_call_like private_class_method :each_call_like_child @@ -654,7 +704,18 @@ def mutating_receiver_allocator_op?(node) sig { params(node: MIR::Node, blk: T.proc.params(arg0: MIR::Node).void).void } def each_mir_expr_child(node, &blk) - return unless node.class.respond_to?(:members) + # T::Struct MIR nodes (RegistryCall and friends) have props, not Struct + # members. Without this they look childless, so an allocating call nested + # under one is never normalized and reaches the checker unhoisted. + unless node.class.respond_to?(:members) + # Only descend into T::Struct nodes whose children can actually be + # replaced -- `replace_t_struct_expr_child!` needs a writable prop or a + # rebuildable wrapper. Yielding a child we cannot replace makes the + # caller hoist a value that stays referenced in place, leaving its + # ErrCleanup without the matching TransferMark. + node.child_exprs.each(&blk) if replaceable_t_struct?(node) + return + end node.class.members.each do |member| value = T.unsafe(node)[member] @@ -1243,6 +1304,10 @@ def normalized_alloc_wrapper_alias?(expr) case expr when MIR::Cast expr.expr.is_a?(MIR::Ident) + when MIR::OptionalUnwrap + # `tmp.?` is a VIEW of a temp that already owns the value. Hoisting it + # into a second owned binding gives the same heap parts two cleanups. + expr.expr.is_a?(MIR::Ident) else false end @@ -1262,7 +1327,10 @@ def mir_consumes_owned_operands?(expr) def replace_mir_expr_child!(parent, old_child, new_child) return if old_child.equal?(new_child) return unless parent.respond_to?(:mir?) && parent.mir? - return unless parent.class.respond_to?(:members) + unless parent.class.respond_to?(:members) + replace_t_struct_expr_child!(parent, old_child, new_child) + return + end parent.class.members.each do |member| value = T.unsafe(parent)[member] @@ -1280,6 +1348,48 @@ def replace_mir_expr_child!(parent, old_child, new_child) nil end + # A T::Struct node whose operands live in an array we can rewrite (the + # RegistryCall/RegistryCallArg shape). Anything else is left alone. + sig { params(node: MIR::Node).returns(T::Boolean) } + def replaceable_t_struct?(node) + node.is_a?(MIR::RegistryCall) + end + + # A T::Struct MIR node holds its operands in props, and an operand may sit + # inside a per-argument wrapper (RegistryCallArg). A `const` prop has no + # writer, so the wrapper is rebuilt around the replacement rather than + # mutated; the array that holds it is the same object either way. + sig { params(parent: MIR::Node, old_child: MIR::Node, new_child: MIR::Node).void } + def replace_t_struct_expr_child!(parent, old_child, new_child) + return unless parent.class.respond_to?(:props) + + parent.class.props.each_key do |prop| + value = T.unsafe(parent).public_send(prop) + if value.equal?(old_child) + next unless parent.respond_to?(:"#{prop}=") + T.unsafe(parent).public_send(:"#{prop}=", new_child) + refresh_ownership_consumption_for_replaced_child!(parent, old_child, new_child) + return + end + next unless value.is_a?(Array) + + value.each_with_index do |item, index| + if item.equal?(old_child) + value[index] = new_child + elsif item.respond_to?(:expr) && item.expr.equal?(old_child) && item.class.respond_to?(:props) + value[index] = item.class.new(**item.class.props.keys.to_h do |key| + [key, key == :expr ? new_child : T.unsafe(item).public_send(key)] + end) + else + next + end + refresh_ownership_consumption_for_replaced_child!(parent, old_child, new_child) + return + end + end + nil + end + MirAggregate = T.type_alias do T.any(T::Array[T.untyped], T::Hash[T.untyped, T.untyped]) end @@ -1482,7 +1592,8 @@ def hoist_cleanup_entry(mir, ast_node) hoist_cleanup_entry(mir.expr, ast_node) when MIR::AsyncPayloadTake, MIR::DirectTenseMap, MIR::Call, MIR::MethodCall, MIR::TryCatch, MIR::Orelse, MIR::IfOptional, MIR::BlockExpr, MIR::Pipeline, - MIR::InlineBc, MIR::RegistryCall, MIR::IndexedStore, MIR::ExternTrampoline, MIR::BgBlock + MIR::InlineBc, MIR::RegistryCall, MIR::IndexedStore, MIR::ExternTrampoline, MIR::BgBlock, + MIR::ShardedMapGet cleanup_entry_for_owned_result(ast_node, alloc: alloc) || typed_cleanup_entry_for_mir_result(mir, alloc: alloc) || cleanup_entry_for_ownership_effect(mir, alloc: alloc) @@ -1605,6 +1716,7 @@ def mir_ident_names(node) end end + private :replace_t_struct_expr_child! private :normalize_allocating_mir_stmt!, :normalize_allocating_result_expr!, :normalize_stmt_child_exprs!, diff --git a/compiler/ruby/mir/lower/pipeline/pipeline_context.rb b/compiler/ruby/mir/lower/pipeline/pipeline_context.rb index e725c00be..0e9cae034 100644 --- a/compiler/ruby/mir/lower/pipeline/pipeline_context.rb +++ b/compiler/ruby/mir/lower/pipeline/pipeline_context.rb @@ -166,15 +166,20 @@ def substitute(node) when AST::BinaryOp then substitute_binary_op(node) when AST::GetField then substitute_get_field(node) when AST::GetIndex then substitute_get_index(node) + when AST::VarDecl then substitute_var_decl(node) when AST::BindExpr then substitute_bind_expr(node) when AST::Assignment then substitute_assignment(node) when AST::UnaryOp then substitute_unary_op(node) + when AST::OptionalUnwrap then substitute_optional_unwrap(node) + when AST::IsA then substitute_is_a(node) when AST::CopyNode, AST::MoveNode, AST::KeepNode, AST::ShareNode substitute_value_wrapper(node) when AST::WithBlock then substitute_with_block(node) when AST::StructLit then substitute_struct_lit(node) when AST::HashLit then substitute_hash_lit(node) when AST::ListLit then substitute_list_lit(node) + when AST::TupleLit then substitute_tuple_lit(node) + when AST::Cast then substitute_cast(node) when AST::BlockExpr then substitute_block_expr(node) when AST::Assert then substitute_assert(node) when AST::IfStatement then substitute_if_statement(node) @@ -274,6 +279,35 @@ def substitute_get_index(node) new_ia end + # A VarDecl carries the declaration's symbol, storage and cleanup stamps, so + # it is rewritten in place: rebuilding it would drop them. The initializer is + # the only place a placeholder can appear. + # An IS_A test rewrites in place: the node carries the annotator's runtime + # payload stamps, and only its subject can hold a placeholder. + sig { params(node: AST::IsA).returns(AST::Node) } + def substitute_is_a(node) + new_left = substitute(node.left) + node.left = new_left unless new_left.equal?(node.left) + node + end + + sig { params(node: AST::OptionalUnwrap).returns(AST::Node) } + def substitute_optional_unwrap(node) + new_target = substitute(node.target) + return node if new_target.equal?(node.target) + + new_unwrap = AST::OptionalUnwrap.new(node.token, new_target) + copy_type_info(node, new_unwrap) + new_unwrap + end + + sig { params(node: AST::VarDecl).returns(AST::Node) } + def substitute_var_decl(node) + new_value = substitute(node.value) + node.value = new_value unless new_value.equal?(node.value) + node + end + sig { params(node: AST::BindExpr).returns(AST::Node) } def substitute_bind_expr(node) new_name = substitute_assignment_target(node.name) @@ -444,17 +478,34 @@ def substitute_list_lit(node) new_ll end + sig { params(node: AST::TupleLit).returns(AST::Node) } + def substitute_tuple_lit(node) + new_items = node.items.map { |item| substitute(item) } + return node if new_items == node.items + + new_tl = AST::TupleLit.new(node.token, new_items, node.storage) + copy_type_info(node, new_tl) + new_tl + end + + sig { params(node: AST::Cast).returns(AST::Node) } + def substitute_cast(node) + new_value = substitute(node.value) + return node if new_value.equal?(node.value) + + new_cast = AST::Cast.new(node.token, new_value, node.target) + copy_type_info(node, new_cast) + new_cast + end + sig { params(node: AST::HashLit).returns(AST::Node) } def substitute_hash_lit(node) pairs = T.let(node.pairs, T::Hash[AST::Node, AST::Node]) new_pairs = T.let({}, T::Hash[AST::Node, AST::Node]) - keys = pairs.keys - index = 0 - while index < keys.length - key = keys.fetch(index) - new_pairs[key] = substitute(pairs.fetch(key)) - index += 1 - end + # Iterate the pairs rather than looking each key back up: the keys are AST + # nodes whose stamps are mutated after insertion, which leaves their hash + # buckets stale and makes `fetch` miss a key that `keys` just handed us. + pairs.each { |key, value| new_pairs[key] = substitute(value) } return node if new_pairs == pairs new_hl = AST::HashLit.new(node.token, new_pairs, node.storage) diff --git a/compiler/ruby/mir/lower/pipeline/pipeline_each_lowerer.rb b/compiler/ruby/mir/lower/pipeline/pipeline_each_lowerer.rb index d6056e590..fdbdb5867 100644 --- a/compiler/ruby/mir/lower/pipeline/pipeline_each_lowerer.rb +++ b/compiler/ruby/mir/lower/pipeline/pipeline_each_lowerer.rb @@ -48,6 +48,7 @@ class PipelineEachLowerer < T::Struct const :lower_sharded_each, T.proc.params(list_node: AST::Node, each_op: AST::EachOp).returns(MIR::ScopeBlock) const :ast_stmts_use_placeholder, T.proc.params(body_stmts: T::Array[AST::Node]).returns(T::Boolean) const :next_index_name, T.proc.returns(String) + const :loop_mark_stmts, T.proc.returns(T::Array[MIR::Emittable]) const :source_alloc_fact, T.proc.params(value: MIR::Node, name: String, type_info: Type).returns(T.nilable([MIR::AllocMark, CleanupEntry])) sig { params(list_node: AST::Node, each_op: AST::EachOp).returns(PipelineEachResult) } @@ -239,10 +240,35 @@ def lower_list_each(list_node, each_op, bc_target:) stmts << MIR::Cleanup.new("__each_src", fact[1]) if fact stmts << MIR::Let.new("__each_items", MIR::ItemsAccess.new(MIR::Ident.new("__each_src"), true), false, nil, nil) - stmts << MIR::ForStmt.new(MIR::Ident.new("__each_items"), "__each_item", list_body_mir, nil) + # Zig rejects an unused capture, and a body that ignores the item is + # ordinary (`list |> EACH { count = count + 1; }`). Vouch for the capture + # rather than predicting whether the body reads it. + stmts << MIR::ForStmt.new(MIR::Ident.new("__each_items"), "__each_item", + [MIR::Suppress.new("__each_item")] + with_iteration_rewind(list_body_mir), nil) MIR::ScopeBlock.new(stmts) end + # An EACH body that allocates frame transients each turn needs the loop's + # per-iteration arena rewind, the same one the SELECT element gets. Without + # it the arena grows for the whole loop and the checker rejects the body's + # iteration-scoped allocations (FRAME_NO_REWIND). + sig { params(body: T::Array[MIR::Emittable]).returns(T::Array[MIR::Emittable]) } + def with_iteration_rewind(body) + return body unless body_frame_transients?(body) + + self.loop_mark_stmts.call.dup + body + end + + sig { params(body: T::Array[MIR::Emittable]).returns(T::Boolean) } + def body_frame_transients?(body) + found = T.let(false, T::Boolean) + boundary = ->(node) { node.is_a?(MIR::BgBlock) || node.is_a?(MIR::LambdaExpr) } + MIR.each_node_until(body, boundary) do |node| + found = true if node.is_a?(MIR::AllocMark) && MIR::Placement.frame?(node.alloc) + end + found + end + sig { params(list_node: AST::Node, each_op: AST::EachOp).returns(MIR::ScopeBlock) } def lower_set_each(list_node, each_op) source_mir = self.visit_mir.call(list_node) @@ -267,12 +293,11 @@ def lower_range_literal_each(list_node, each_op) end_mir = self.visit_mir.call(range.finish) end_expr = range.inclusive ? MIR::BinOp.new("+", end_mir, MIR::Lit.new("1")) : end_mir range_body_mir = self.visit_body_with_placeholder.call(each_op.body, "__each_item") - capture_name = self.ast_stmts_use_placeholder.call(each_op.body) ? "__each_item" : "_" MIR::ForStmt.new( MIR::IterRange.new(start_mir, end_expr, :i64), - capture_name, - range_body_mir, + "__each_item", + [MIR::Suppress.new("__each_item")] + range_body_mir, nil, ) end diff --git a/compiler/ruby/mir/lower/pipeline/pipeline_host.rb b/compiler/ruby/mir/lower/pipeline/pipeline_host.rb index 81eb1a8e3..f421b004f 100644 --- a/compiler/ruby/mir/lower/pipeline/pipeline_host.rb +++ b/compiler/ruby/mir/lower/pipeline/pipeline_host.rb @@ -77,6 +77,7 @@ def build_plan_builder sig { returns(PipelineScalarLowerer) } def build_scalar_lowerer PipelineScalarLowerer.new( + loop_mark_stmts: -> { @lowering_bridge.pipeline_iteration_loop_marks }, visit_expr: ->(_list_node, expr_node, placeholder) { with_pipeline_context(placeholder: placeholder) { visit_mir(expr_node) } }, @@ -192,6 +193,7 @@ def build_each_lowerer lower_each_range: ->(source_node, stages, each_op) { lower_each_range(source_node, stages, each_op) }, lower_sharded_each: ->(list_node, each_op) { lower_sharded_each(list_node, each_op) }, ast_stmts_use_placeholder: ->(body_stmts) { ast_stmts_use_placeholder?(body_stmts) }, + loop_mark_stmts: -> { @lowering_bridge.pipeline_iteration_loop_marks }, source_alloc_fact: ->(value, name, type_info) { fact = @lowering_bridge.pipeline_alloc_mark_fact( value, name, fallback_alloc: :heap, type_info: type_info, diff --git a/compiler/ruby/mir/lower/pipeline/pipeline_scalar_lowerer.rb b/compiler/ruby/mir/lower/pipeline/pipeline_scalar_lowerer.rb index 38d47e2c7..3ac8aab34 100644 --- a/compiler/ruby/mir/lower/pipeline/pipeline_scalar_lowerer.rb +++ b/compiler/ruby/mir/lower/pipeline/pipeline_scalar_lowerer.rb @@ -26,6 +26,7 @@ class PipelineScalarLowerer < T::Struct const :visit_expr, T.proc.params(list_node: AST::Node, expr_node: AST::Node, placeholder: String).returns(MIR::Node) const :pipeline_block, T.proc.params(list_node: AST::Node, blk: T.proc.params(items: String, label: String).returns(T::Array[MIR::Emittable])).returns(MIR::BlockExpr) const :transpile_type, T.proc.params(type_info: PipelineTypeInput).returns(String) + const :loop_mark_stmts, T.proc.returns(T::Array[MIR::Emittable]) sig { params(site: PipelineSite, op: PipelineMaterializedScalarOp).returns(MIR::BlockExpr) } def lower(site, op) @@ -58,7 +59,7 @@ def lower_count(site, count_node) self.pipeline_block.call(list_node, lambda do |items, label| [ MIR::Let.new("count_result", MIR::Lit.new("0"), true, Type.new("i64"), nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::IfStmt.new(pred_mir, [ MIR::Set.new(MIR::Ident.new("count_result"), MIR::BinOp.new("+", MIR::Ident.new("count_result"), MIR::Lit.new("1"))), @@ -78,7 +79,7 @@ def lower_sum(site, sum_node) self.pipeline_block.call(list_node, lambda do |items, label| [ MIR::Let.new("sum_result", MIR::Lit.new(zero), true, result_type, nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::Set.new(MIR::Ident.new("sum_result"), MIR::BinOp.new("+", MIR::Ident.new("sum_result"), expr_mir)), ], nil), @@ -95,7 +96,7 @@ def lower_average(site, avg_node) [ MIR::Let.new("avg_sum", MIR::Lit.new("0"), true, Type.new("f64"), nil), MIR::Let.new("avg_count", MIR::FieldGet.new(MIR::Ident.new(items), "len"), false, nil, nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::Set.new(MIR::Ident.new("avg_sum"), MIR::BinOp.new("+", MIR::Ident.new("avg_sum"), expr_mir)), ], nil), @@ -125,7 +126,7 @@ def lower_min(site, min_node) nil), MIR::Let.new("min_result", MIR::TypeSentinel.new(:max, zig_type), true, result_type, nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::Let.new("min_val", expr_mir, false, nil, nil), MIR::IfStmt.new( MIR::BinOp.new("<", MIR::Ident.new("min_val"), MIR::Ident.new("min_result")), @@ -154,7 +155,7 @@ def lower_max(site, max_node) nil), MIR::Let.new("max_result", sentinel, true, result_type, nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::Let.new("max_val", expr_mir, false, nil, nil), MIR::IfStmt.new( MIR::BinOp.new(">", MIR::Ident.new("max_val"), MIR::Ident.new("max_result")), @@ -173,7 +174,7 @@ def lower_any(site, any_node) self.pipeline_block.call(list_node, lambda do |items, label| [ MIR::Let.new("any_result", MIR::Lit.new("false"), true, nil, nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::IfStmt.new(pred_mir, [ MIR::Set.new(MIR::Ident.new("any_result"), MIR::Lit.new("true")), MIR::BreakStmt.new(nil, nil), @@ -191,7 +192,7 @@ def lower_all(site, all_node) self.pipeline_block.call(list_node, lambda do |items, label| [ MIR::Let.new("all_result", MIR::Lit.new("true"), true, nil, nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::IfStmt.new(MIR::UnaryOp.new("!", pred_mir), [ MIR::Set.new(MIR::Ident.new("all_result"), MIR::Lit.new("false")), MIR::BreakStmt.new(nil, nil), @@ -212,7 +213,7 @@ def lower_find(site, find_node) MIR::Let.new("find_result", MIR::Undef.new(nil), true, Type.new(elem_zig_type), nil), MIR::Let.new("find_found", MIR::Lit.new("false"), true, nil, nil), - MIR::ForStmt.new(MIR::Ident.new(items), "it", [ + scalar_loop(MIR::Ident.new(items), "it", [ MIR::Let.new("find_matches", pred_mir, false, nil, nil), MIR::IfStmt.new(MIR::Ident.new("find_matches"), [ MIR::Set.new(MIR::Ident.new("find_result"), MIR::Ident.new("it")), @@ -233,4 +234,29 @@ def lower_find(site, find_node) def visit_pipeline_expr_mir(list_node, expr_node, placeholder = "it") self.visit_expr.call(list_node, expr_node, placeholder) end + + # A scalar pipeline accumulates into a scalar, so anything its body allocates + # on the frame dies with the iteration and the loop can rewind -- the same + # per-iteration rewind a SELECT element gets. Without it the arena grows for + # the whole loop (FRAME_NO_REWIND). + sig do + params(iter: MIR::Emittable, capture: String, body: T::Array[MIR::Emittable], mark: T.nilable(T::Boolean)) + .returns(MIR::ForStmt) + end + def scalar_loop(iter, capture, body, mark = nil) + MIR::ForStmt.new(iter, capture, with_iteration_rewind(body), mark) + end + + sig { params(body: T::Array[MIR::Emittable]).returns(T::Array[MIR::Emittable]) } + def with_iteration_rewind(body) + found = T.let(false, T::Boolean) + boundary = ->(node) { node.is_a?(MIR::BgBlock) || node.is_a?(MIR::LambdaExpr) } + MIR.each_node_until(body, boundary) do |node| + found = true if node.is_a?(MIR::AllocMark) && MIR::Placement.frame?(node.alloc) + end + return body unless found + + self.loop_mark_stmts.call.dup + body + end + end diff --git a/compiler/ruby/mir/lowering/concurrency.rb b/compiler/ruby/mir/lowering/concurrency.rb index f4a615911..0e5b040be 100644 --- a/compiler/ruby/mir/lowering/concurrency.rb +++ b/compiler/ruby/mir/lowering/concurrency.rb @@ -221,7 +221,9 @@ def with_stream_body_context(local_stream, is_inf, close_label: nil, inherited_a capture_state.current_stream_local = prev_stream_local capture_state.current_stream_is_inf = prev_stream_is_inf capture_state.current_stream_close_label = prev_close_label - capture_state.current_fsm_inherited_alloc_names = T.must(prev_inherited_alloc_names) + # `ensure` may run before line 213 assigns (an earlier statement raised); + # the prop setter rejects nil, so only restore a snapshot that was taken. + capture_state.current_fsm_inherited_alloc_names = prev_inherited_alloc_names unless prev_inherited_alloc_names.nil? end sig { params(caps: FiberCtxBuilder::Result, analysis: T.nilable(CapabilityHelper::CaptureAnalysis), receiver: String, close_plans: T::Hash[String, Schemas::ResourceClosePlan]).returns(T::Array[MIR::Stmt]) } diff --git a/compiler/ruby/mir/lowering/control_flow.rb b/compiler/ruby/mir/lowering/control_flow.rb index 146228544..318a4d558 100644 --- a/compiler/ruby/mir/lowering/control_flow.rb +++ b/compiler/ruby/mir/lowering/control_flow.rb @@ -169,6 +169,25 @@ def lower_runtime_is_a_if(node, condition) with_pending(subject_pending, MIR::IfStmt.new(cond, then_body, else_body)) end + # A payload binding occupies its Zig name for the rest of the branch. Nested + # MATCHes that bind the same name (`item` inside an arm that already bound + # `item`) would redeclare it, which Zig rejects -- so a colliding binding + # takes the same `_L` rename a colliding local declaration takes. + sig { params(binding: String, decl: T.untyped, line: T.nilable(Integer)).returns(String) } + def payload_binding_name(binding, decl, line) + T.bind(self, MIRLowering) rescue nil + safe = zig_safe_name(binding) + if function_state.alloc_marked_names.key?(safe) + suffix = line ? "_L#{function_relative_line(line)}" : "_#{lowering_counters.next_tmp_id}" + safe = zig_safe_name("#{binding}#{suffix}") + end + function_state.alloc_marked_names[safe] = true + # Key the rename by the declaration's identity, not by name: a name-keyed + # map would keep pointing at the inner binding after the nested MATCH ends. + function_state.decl_zig_names[decl.object_id] = safe if decl + safe + end + sig { params(condition: AST::IsA, subject: MIR::Emittable, variant: String).returns(MatchBody) } def runtime_is_a_payload_bindings(condition, subject, variant) binding = condition.binding @@ -177,7 +196,8 @@ def runtime_is_a_payload_bindings(condition, subject, variant) payload = T.let(MIR::UnionPayloadGet.new(subject, variant), MIR::Emittable) payload = MIR::Deref.new(payload) if condition.runtime_indirect_payload_as is_mutable = condition.left.is_a?(AST::Identifier) && condition.left.was_moved == true - [MIR::Let.new(binding, payload, is_mutable, nil, "_ = &#{binding};")] + safe_binding = payload_binding_name(binding.to_s, condition, condition.line) + [MIR::Let.new(safe_binding, payload, is_mutable, nil, "_ = &#{safe_binding};")] end sig { params(node: AST::IfBind).returns(MIR::IfBindStmt) } @@ -678,15 +698,21 @@ def for_each_loop_stmt(node, plan) ) loop_stmt = MIR::ScopeBlock.new([iter_init, while_stmt]) else - is_field_access = node.collection.is_a?(AST::GetField) is_param = node.collection.is_a?(AST::Identifier) && current_function_param_name?(node.collection.name) # list_collection? covers T[N]@list (fixed capacity ArrayList) in addition to # T[]@list (dynamic). Both map to std.ArrayListUnmanaged and require .items. + # A `[]T@list` is an ArrayList and iterates its `.items` whether it is a + # local or reached through a field. Only a `T[]` FIELD is a plain slice, + # and slices iterate directly. + is_field_access = node.collection.is_a?(AST::GetField) is_arraylist = (ct.list_collection? || (ct.array? && ct.dynamic?)) && - !ct.string? && !is_param && !is_field_access + !ct.string? && !is_param && + !(is_field_access && ct.slice_shaped_field_array?) iter = if is_arraylist MIR::ListItems.new(coll) + elsif is_field_access + coll elsif is_param # @list params are anytype — could be ArrayList (TAKES) or slice (borrow, # via .items at call site). MIR::ItemsAccess(safe: true) emits a comptime @@ -694,8 +720,6 @@ def for_each_loop_stmt(node, plan) # zero runtime overhead. Defer container shape to the runtime/comptime # layer instead of re-deriving from "is this a param?". MIR::ItemsAccess.new(coll, true) - elsif is_field_access && ct.dynamic_field_array? - coll else MIR::AddressOf.new(coll) end @@ -856,7 +880,9 @@ def union_if_chain_payload_bindings(match_case, subject, variant, is_mutable) payload = T.let(MIR::UnionPayloadGet.new(subject, variant.to_s), MIR::Emittable) payload = MIR::Deref.new(payload) if match_case.indirect_payload_as if match_case.binding - return [MIR::Let.new(T.must(match_case.binding), payload, is_mutable, nil, "_ = &#{match_case.binding};")] + safe_binding = payload_binding_name(T.must(match_case.binding).to_s, match_case, + match_case.respond_to?(:line) ? match_case.public_send(:line) : nil) + return [MIR::Let.new(safe_binding, payload, is_mutable, nil, "_ = &#{safe_binding};")] end destructure = match_case.destructure @@ -1144,6 +1170,10 @@ def lower_return(node) plan = return_lowering_plan(node) value = finalize_return_value(node, plan.value) + # `RETURN panic("...")` has no value to return: the expression itself is + # the terminator, and `return @panic(...)` is unreachable code. + return value if value && Hoist.noreturn_value?(node.value) + # Tail call optimization: convert self-recursive return to @call(.always_tail, ...) # Disabled in debug mode (stage2 Zig backend doesn't support always_tail reliably) if value.is_a?(MIR::Call) && tail_call_return?(value) diff --git a/compiler/ruby/mir/lowering/expressions.rb b/compiler/ruby/mir/lowering/expressions.rb index 1f8ae5999..4e5d4ac21 100644 --- a/compiler/ruby/mir/lowering/expressions.rb +++ b/compiler/ruby/mir/lowering/expressions.rb @@ -413,8 +413,9 @@ def lower_binary_op(node) # String concat (2-part) uses std.mem.concat if node.string_concat - left = hoist_alloc(T.cast(lower(node.left), MIR::Node), node.left) - right = hoist_alloc(T.cast(lower(node.right), MIR::Node), node.right) + # Concat consumes bytes; a Symbol operand widens to the bytes behind it. + left = hoist_alloc(widen_symbol_to_bytes(T.cast(lower(node.left), MIR::Node), node.left), node.left) + right = hoist_alloc(widen_symbol_to_bytes(T.cast(lower(node.right), MIR::Node), node.right), node.right) alloc = alloc_for_node(node) return MIR::ConcatStr.new([left, right], alloc, nil) end @@ -457,11 +458,12 @@ def type_value_zig_name(name) sig { params(node: AST::GetField).returns(String) } def dotted_type_value_zig_name(node) + T.bind(self, MIRLowering) rescue nil if node.target.is_a?(AST::Identifier) namespace = T.cast(node.target, AST::Identifier).name return type_value_zig_name(node.field.to_s) if namespace == "AST" - return "#{namespace}.#{node.field}" + return "#{zig_module_alias(namespace)}.#{node.field}" end Kernel.raise "MIRLowering: unsupported dotted type expression #{node.inspect}" @@ -557,11 +559,14 @@ def string_comparison_operator(op) sig { params(facts: BinaryOperandFacts).returns(T.nilable(BinaryOperationPlan)) } def classify_optional_binary_comparison(facts) return nil unless OPTIONAL_COMPARISON_OPS.include?(facts.op) - return nil unless facts.left_type.optional? != facts.right_type.optional? + return nil unless facts.left_type.optional? || facts.right_type.optional? + + both_optional = facts.left_type.optional? && facts.right_type.optional? + return nil if both_optional && !(facts.op == :EQ || facts.op == :NEQ) optional_side = facts.left_type.optional? ? OptionalOperandSide::Left : OptionalOperandSide::Right payload_type = optional_side == OptionalOperandSide::Left ? facts.right_type : facts.left_type - return nil if payload_type.resolved == :NIL + return nil if !both_optional && payload_type.resolved == :NIL BinaryOperationPlan.new( kind: :optional_comparison, @@ -668,12 +673,23 @@ def emit_optional_comparison_plan(plan) capture_ref = MIR::Ident.new(capture) optional_source = optional_side == OptionalOperandSide::Left ? facts.left : facts.right then_expr = emit_optional_comparison_then_expr(facts, optional_side, capture_ref) - else_expr = MIR::Lit.new(facts.op == :NEQ ? "true" : "false") + else_expr = absent_optional_comparison_result(facts, optional_side) result = MIR::IfOptional.new(optional_source, capture, then_expr, else_expr) result.result_type = Type.new(:Bool) result end + # The unwrapped side turned out to be absent. Against a payload that answer is + # fixed; against another optional it depends on whether that one is absent too. + sig { params(facts: BinaryOperandFacts, optional_side: OptionalOperandSide).returns(MIR::Node) } + def absent_optional_comparison_result(facts, optional_side) + other = optional_side == OptionalOperandSide::Left ? facts.right : facts.left + other_type = optional_side == OptionalOperandSide::Left ? facts.right_type : facts.left_type + return MIR::Lit.new(facts.op == :NEQ ? "true" : "false") unless other_type.optional? + + MIR::BinOp.new(facts.op == :NEQ ? "!=" : "==", other, MIR::Lit.new("null")) + end + sig { params(facts: BinaryOperandFacts, optional_side: OptionalOperandSide, capture_ref: MIR::Ident).returns(MIR::Node) } def emit_optional_comparison_then_expr(facts, optional_side, capture_ref) inner_type = optional_side == OptionalOperandSide::Left ? T.must(facts.left_type.wrapped_type) : T.must(facts.right_type.wrapped_type) @@ -1124,7 +1140,21 @@ def lower_or_else(node) fallback_type = or_fallback_expected_type(node) right = lower_scoped do with_expected_type(fallback_type) do - materialize_or_fallback_value(T.cast(lower(node.right), MIR::Node), node.right) + # The fallback coerces to the MERGE type here, where its stamp is + # still in hand -- placement dupes each branch and cannot widen a bare + # MIR value. A symbol fallback merging into a String widens to its + # bytes; a string LITERAL merging into a symbol wraps as a handle, + # which is sound only for literals (rodata is immortal; wrapping an + # owned String would orphan its cleanup). + fallback_mir = T.cast(lower(node.right), MIR::Node) + merge_type = Type.from_node!(node, context: "OR_ELSE merge type") + if merge_type.byte_string? + fallback_mir = widen_symbol_to_bytes(fallback_mir, node.right) + elsif merge_type.symbol? && node.right.is_a?(AST::Literal) && T.cast(node.right, AST::Literal).type == :STRING + fallback_mir = MIR::Call.new("CheatLib.symbolOf", [fallback_mir], false, false, + MIR::CallableContract.no_ownership(1)) + end + materialize_or_fallback_value(fallback_mir, node.right) end end @@ -1192,7 +1222,9 @@ def materialize_or_fallback_value(value, ast_node) return value unless ti.string? || ti.recursive_cleanup_shape?(T.unsafe(mir_schema_lookup)) || ti.needs_cleanup?(T.unsafe(mir_schema_lookup)) alloc = function_state.current_decl_alloc || :heap - copied = MIR::DeepCopy.new(value, ti.zig_type, nil, :full_value, alloc) + # An Rc/Arc fallback value is retained, never structurally copied. + copied = retain_handle_for_destination(value, ti) || + MIR::DeepCopy.new(value, ti.zig_type, nil, :full_value, alloc) hoist_alloc(copied, ast_node, err_cleanup: false) end @@ -2166,8 +2198,10 @@ def lower_struct_lit(node) field_alloc = mir_owned_alloc(field_value) lowered = hoist_alloc(field_value, field_node, err_cleanup: true) if expected_ft && recursive_field_copy_required?(expected_ft, field_node, field_alloc, field_sink_alloc) - hoist_alloc(MIR::DeepCopy.new(lowered, expected_ft.zig_type, nil, :full_value, field_sink_alloc), - field_node, err_cleanup: true) + # An Rc/Arc field is retained, never structurally copied. + copy = retain_handle_for_destination(lowered, expected_ft) || + MIR::DeepCopy.new(lowered, expected_ft.zig_type, nil, :full_value, field_sink_alloc) + hoist_alloc(copy, field_node, err_cleanup: true) else lowered end @@ -2185,15 +2219,31 @@ def lower_struct_lit(node) # @boxed field: hoist HeapCreate to a named temp so it is a Let-init, # not an anonymous sub-expression (INV-H). if v.needs_heap_create - zig_t = transpile_type(v.full_type!.resolved.to_s) + field_ti = v.full_type!(context: "indirect struct field allocation") + # `?T@boxed` is an OPTIONAL POINTER: the box holds the payload and + # absence is the null pointer. Boxing the optional itself allocates a + # cell for `?T` and hands back `*?T`, which is not the field's type -- + # and it allocates even when there is nothing to hold. + payload_ti = field_ti.optional? ? T.must(field_ti.wrapped_type) : field_ti + zig_t = transpile_type(payload_ti.resolved.to_s) temp = "__ind_#{lowering_counters.next_block_expr_id}_#{k}" - hc = T.cast(with_ownership_consumption_for_value( - MIR::HeapCreate.new(zig_t, val, :heap, "blk_#{k}"), + boxed = if field_ti.optional? + capture = "__box_some_#{lowering_counters.next_tmp_id}" + MIR::IfOptional.new( + val, capture, + MIR::HeapCreate.new(zig_t, MIR::Ident.new(capture), :heap, "blk_#{k}"), + MIR::Lit.new("null"), + ) + else + MIR::HeapCreate.new(zig_t, val, :heap, "blk_#{k}") + end + hc = with_ownership_consumption_for_value( + boxed, val, field_node, "MIR::HeapCreate", target_alloc: :heap, - ), MIR::HeapCreate) + ) hoisted.concat(MIR::BindingMaterialization.new( name: temp, expr: hc, @@ -2204,9 +2254,16 @@ def lower_struct_lit(node) ).statements) # errdefer cleans this field if a later allocation (another field or # the outer struct pointer) fails. - hoisted << MIR::ErrDeferStmt.new( - MIR::DestroyPtr.new(MIR::Ident.new(temp), :heap) - ) + destroy = T.let(MIR::DestroyPtr.new(MIR::Ident.new(temp), :heap), MIR::Node) + if field_ti.optional? + capture = "__box_free_#{lowering_counters.next_tmp_id}" + destroy = MIR::IfOptional.new( + MIR::Ident.new(temp), capture, + MIR::DestroyPtr.new(MIR::Ident.new(capture), :heap), + MIR::ScopeBlock.new([]), + ) + end + hoisted << MIR::ErrDeferStmt.new(destroy) val = MIR::Ident.new(temp) end { name: k.to_s, value: val, alloc: field_sink_alloc } @@ -2484,7 +2541,7 @@ def aggregate_field_sink_alloc(_field_type, value, aggregate_alloc) sig { params(node: AST::StringConcat).returns(MIR::ConcatStr) } def lower_string_concat(node) T.bind(self, MIRLowering) rescue nil - parts = node.parts.map { |p| hoist_alloc(lower(p), p) } + parts = node.parts.map { |p| hoist_alloc(widen_symbol_to_bytes(lower(p), p), p) } alloc = alloc_for_node(node) MIR::ConcatStr.new(parts, alloc, runtime_binding_name) end @@ -2509,7 +2566,12 @@ def lower_block_expr(node) # block. Lowering a nil result crashed even the error path (nil.token). return MIR::ScopeBlock.new(body) if node.result.nil? - result = lower(node.result) + # The tail expression's materializations belong INSIDE this block: they can + # reference locals the block declares, and lower() would otherwise leave + # them in function_state.pending_stmts for the enclosing statement to + # flush -- above the block, past the declarations they use. + result, result_hoists = lower_head { lower(node.result) } + body.concat(result_hoists) if transfer_name cleanup = body.find do |stmt| (stmt.is_a?(MIR::Cleanup) || stmt.is_a?(MIR::ErrCleanup)) && stmt.name.to_s == transfer_name @@ -2520,7 +2582,13 @@ def lower_block_expr(node) end end body << MIR::BreakStmt.new(label, result) - MIR::BlockExpr.new(label, body) + block = MIR::BlockExpr.new(label, body) + # The tail expression is already annotated, so stamp the block's result + # type here; otherwise hoisting re-derives it from the MIR body shape and + # fails on anything its shape table doesn't enumerate (tuple literals, + # blocks with more than one AllocMark, ...). + block.result_type = Type.from_node!(node.result, context: "block expression result") + block end sig { params(node: AST::RangeLit).returns(MIR::RangeLit) } @@ -2638,6 +2706,13 @@ def try_lower_equality_assert(node) helper, extra_args = pick_equality_helper(left, right) return nil unless helper + # expectEqualStrings takes []const u8, so a Symbol operand crosses a String + # coercion boundary here exactly as it does at a call or a cast. + if helper == "expectEqualStrings" + left_mir = widen_symbol_to_bytes(left_mir, left) + right_mir = widen_symbol_to_bytes(right_mir, right) + end + # Argument order matches the Zig stdlib convention: expected # first, actual second. CLEAR doesn't distinguish, so we use # left=expected, right=actual. @@ -2776,6 +2851,9 @@ def lower_copy(node) if fresh_copy_constructor?(node.value) return source end + # COPY of a NoReturn expression has nothing to duplicate: binding it to a + # copy temp emits `const __copy_src = @panic(...)`, which is unreachable. + return source if Hoist.noreturn_value?(node.value) # A payload-free union constructor (for example `Value.Nil`) is already a # fresh value and contains no storage to duplicate. Auto-COPY may wrap it # when it appears as an owned fallback; lowering that wrapper as a full @@ -2842,6 +2920,16 @@ def lower_copy(node) # using that payload destination here asks dupeValue to clone T from ?T # (or !T), which is both type-incorrect and loses wrapper semantics. copy_ti = (ti.optional? || ti.error_union?) ? ti : dst_ti + # A carrier DESTINATION is built around the copied payload by the + # enclosing wrap; COPY duplicates the plain value, never the handle. + # This has to win over the optional/borrowed spellings below, which + # would otherwise render the destination's Rc(T)/Arc(T). + dst_payload = dst_ti.non_optional_type + source_payload = ti.non_optional_type + if dst_payload.any_rc? && !source_payload.any_rc? + return MIR::DeepCopy.new(source, transpile_type(source_payload), nil, :full_value, alloc, + MIR::DeepCopy.copy_shape_for_zig_type(transpile_type(source_payload)), source_payload) + end copy_zig = if lifecycle.copy_strategy == :generic # Generic/projection values already have their concrete Zig type at # comptime. Re-rendering the unresolved CLEAR shape here can leak @@ -2858,6 +2946,12 @@ def lower_copy(node) # Borrowing is represented as an implementation pointer, not as part # of the copied value's logical type. COPY owns the pointee. bare_zig_type(dst_ti) + elsif dst_ti.indirect? && !ti.indirect? + # Same reasoning as the Rc case below: the destination's box is + # created by the enclosing placement step, which allocates the cell + # and stores the payload into it. COPY duplicates that payload; + # typing it `*T` tells dupeValue the value is already boxed. + transpile_type(ti) elsif dst_ti.any_rc? && !ti.any_rc? # The destination capability is created by the enclosing declaration's # CapWrap. COPY must duplicate the plain payload that will be placed in @@ -3256,6 +3350,7 @@ def generic_type_arg_zig(type) Type.new(type).zig_type end + private :absent_optional_comparison_result private :emit_optional_comparison_then_expr private :aggregate_field_wants_dynamic_slice? diff --git a/compiler/ruby/mir/lowering/functions.rb b/compiler/ruby/mir/lowering/functions.rb index 2ea4647e9..8f1af504e 100644 --- a/compiler/ruby/mir/lowering/functions.rb +++ b/compiler/ruby/mir/lowering/functions.rb @@ -94,6 +94,11 @@ class StdlibCallArgFact < T::Struct const :takes, T::Boolean const :coerce_type, T.nilable(Symbol) const :sink_type, T.nilable(Type) + # The registry declares this argument as a plain String ([]const u8), so a + # Symbol argument must widen to its bytes -- the same coercion user calls + # perform at cross_boundary_arg. False for :Any and container positions, + # whose element types (a Set of symbols, say) take the handle itself. + const :declared_byte_string, T::Boolean, default: false sig { params(arg_zig: String).returns(String) } def coerce_zig(arg_zig) @@ -579,7 +584,13 @@ def function_lowering_context(node, final_type, return_type_node, fn_needs_rt, p heap_carry_return_vars: typed_name_set(node.heap_carry_return_vars), returned_names: collect_fn_returned_names(node.body), snapshot_types: has_catch ? typed_name_set(node.snapshot_types) : Set.new, - fn_alloc_marked_names: {}, + # A local that shadows a parameter (or the runtime handle) is legal CLEAR + # and legal Ruby, but Zig rejects any shadowing. Seeding the name table + # with the parameters makes var_decl_safe_name disambiguate such a local + # the same way it already disambiguates two same-named locals. + fn_alloc_marked_names: node.params.each_with_object({ "rt" => true }) { |p, acc| + acc[zig_safe_name(p.name.to_s)] = true + }, lowered_alloc_names: Set.new, lowered_guarded_cleanup_names: Set.new, decl_zig_name_map: {}, @@ -1171,7 +1182,18 @@ def cross_boundary_arg(arg, a, callee_param, callee_param_type, callee_sig, idx) # callee per carrier. Never detach a handle to a plain payload here -- that # would destroy the retained identity the contract exists to preserve. The # callee's universal comptime cleanup releases whatever carrier arrived. - if callee_param&.takes && callee_param.carrier_contract == :monomorphic + # A Symbol reaching a plain-String parameter widens to the bytes behind + # it, the same borrow CAST and placement perform: Zig will not coerce the + # distinct handle type. TAKES receives an owned COPY instead -- the callee + # frees its parameter, and interned bytes are nobody's to free. Skipped + # for MONOMORPHIC params, which thread the caller's carrier unchanged. + if ti&.symbol? && callee_param_type.byte_string? && + callee_param&.carrier_contract != :monomorphic + widened = MIR::FieldGet.new(arg, "bytes") + return callee_param&.takes ? MIR::DupeSlice.new(widened, :heap) : widened + end + +if callee_param&.takes && callee_param.carrier_contract == :monomorphic return MIR::AddressOf.new(arg) if wants_ptr?(a, ti, callee_param, callee_param_type, callee_sig, idx) return arg end @@ -1283,11 +1305,35 @@ def materialize_mutable_call_temporary(arg, ast_arg) hoisted = hoist_alloc(arg, ast_arg, mutable: true) return hoisted unless hoisted.equal?(arg) + # Copying an owned temp into the mutable slot splits ownership: a MUTABLE + # param writes its result back into the slot while the cleanup stays on the + # original binding, so the callee's value leaks and the original is freed + # twice. The temp was hoisted for this call alone -- make it addressable + # rather than copying it. + owned_temp = pending_owned_let(arg) + if owned_temp + owned_temp.mutable = true + return arg + end + name = "__mutable_arg_#{lowering_counters.next_tmp_id}" function_state.pending_stmts << MIR::Let.new(name, arg, true, nil, nil) MIR::Ident.new(name) end + # The Let this statement just hoisted for `arg`, when this function owns it. + sig { params(arg: MIR::Node).returns(T.nilable(MIR::Let)) } + def pending_owned_let(arg) + T.bind(self, MIRLowering) rescue nil + return nil unless arg.is_a?(MIR::Ident) + + name = arg.name.to_s + T.cast( + function_state.pending_stmts.find { |stmt| stmt.is_a?(MIR::Let) && stmt.name.to_s == name }, + T.nilable(MIR::Let), + ) + end + sig { params(callee_param: T.nilable(AST::Param), moved_arg: T::Boolean, ti: Type, callee_param_type: Type).returns(T::Boolean) } def owned_slice_argument_required?(callee_param, moved_arg, ti, callee_param_type) !!(callee_param&.takes && moved_arg && ti.direct_indexable_collection? && @@ -1452,6 +1498,9 @@ def stdlib_call_facts(node) takes: ownership.takes?(index), coerce_type: stdlib_coerce_type(param.type), sink_type: stdlib_sink_type_for_arg(receiver_type, index, ownership.takes?(index)), + # The registry declares plain byte strings as :String; the interned + # handle is only ever a RETURN sync (`symbol()`), never a param spelling. + declared_byte_string: stdlib_coerce_type(param.type) == :String, ) end StdlibCallFacts.new(args: facts, ownership: ownership) @@ -1770,6 +1819,7 @@ def managed_handle_materialized_for_plain_takes?(ast_arg, contract, idx) return false unless a.is_a?(AST::Identifier) !!(current_function_collection_param?(a.name) || with_alias_pointer_shaped?(a) || + capture_state.current_lambda_pointer_params.include?(a.name.to_s) || capture_state.current_bg_pointer_captures&.include?(a.name)) end @@ -2182,7 +2232,11 @@ def call_type_owned_return?(ti, sig_obj) union_schema = union_schemas[schema_name] if union_schema variants = union_schema.respond_to?(:variants) ? union_schema.variants : {} - return variants.any? { |_, variant_type| Type.variant_has_heap?(variant_type) } + # variant_has_heap? only sees a bare heap pointer in the variant slot. A + # variant that names a struct owning heap fields is just as owned, and + # the recursive shape check below answers that -- so fall through + # instead of returning false. + return true if variants.any? { |_, variant_type| Type.variant_has_heap?(variant_type) } end ti.ownership_bearing?(T.unsafe(mir_schema_lookup)) || @@ -2461,6 +2515,9 @@ def materialize_stdlib_arguments(mir_args, stdlib_facts, ownership_facts, sink_a materialized_args = mir_args.dup stdlib_facts.args.each do |arg_fact| index = arg_fact.index + if arg_fact.declared_byte_string + materialized_args[index] = widen_symbol_to_bytes(T.must(materialized_args[index]), arg_fact.ast_arg) + end next unless ownership_facts.takes?(index) placed_arg = place_value_for_destination( @@ -2642,10 +2699,9 @@ def build_extern_trampoline_call(node) id = lowering_counters.next_extern_id alloc_kind = node.respond_to?(:extern_effects) ? node.extern_effects&.dig(:alloc) : nil mod_alias = T.unsafe(node).module_alias if node.respond_to?(:module_alias) - mod_alias = nil unless mod_alias.is_a?(String) source = node.respond_to?(:extern_source) ? node.extern_source : nil mod_alias = nil if source&.abi == :c - mod_alias = zig_module_alias(mod_alias) if mod_alias + mod_alias = T.let(mod_alias ? zig_module_alias(mod_alias) : nil, T.nilable(String)) # Separate comptime type args (full_type == :Type) from runtime args. # Comptime args can't be struct fields; the emitter renders them directly @@ -2776,6 +2832,19 @@ def lower_c_abi_callback_arg(arg, param, source) # Lambda # ================================================================ + # Does the lambda's tail value come out of a pipeline? Its placement belongs + # to escape analysis, so lowering leaves it alone. + sig { params(expr: AST::Node).returns(T::Boolean) } + def lambda_tail_pipeline?(expr) + node = T.let(expr, T.untyped) + while node.is_a?(AST::BlockExpr) || node.is_a?(AST::Cast) + node = node.is_a?(AST::BlockExpr) ? node.result : node.value + end + return false unless node.is_a?(AST::BinaryOp) + + node.smooth? == true + end + sig { params(node: AST::LambdaLit).returns(MIR::LambdaExpr) } def lower_lambda(node) T.bind(self, MIRLowering) rescue nil @@ -2790,6 +2859,10 @@ def lower_lambda(node) pt_obj = p_type.is_a?(Type) ? p_type : (Type.new(p_type) rescue nil) pp = !!(pt_obj && (pt_obj.respond_to?(:needs_pointer_passing?) && pt_obj.needs_pointer_passing? || (p.mutable && pt_obj.respond_to?(:list_collection?) && pt_obj.list_collection?))) + # A MUTABLE lambda parameter is passed by pointer, exactly like a MUTABLE + # parameter of a named function. + pp ||= p.mutable == true + type_str = "*#{type_str}" if p.mutable && !type_str.start_with?("*") MIR::Param.new(p.name, type_str, pp) }, T::Array[MIR::Param]) @@ -2802,24 +2875,45 @@ def lower_lambda(node) params_list.each { |p| body_mir << MIR::Suppress.new(p.name) } body_nodes = AST.lambda_body_nodes(node.body) prefix_nodes = body_nodes[0...-1] || [] - body_mir.concat(lower_body(prefix_nodes)) return_expr = T.must(body_nodes.last) lambda_return = AST::ReturnNode.new(return_expr.respond_to?(:token) ? T.unsafe(return_expr).token : nil, return_expr) + previous_pointer_params = capture_state.current_lambda_pointer_params + capture_state.current_lambda_pointer_params = + params_list.select { |p| p.mutable == true }.map { |p| p.name.to_s }.to_set + # Inside the lambda the runtime is its own `_rt` parameter; the enclosing + # function's `rt` is not in scope there (Zig rejects the reference). + body_mir.concat(runtime_state.with_rt_name("_rt") { lower_body(prefix_nodes) }) # Capture the return expression's pending hoists INSIDE the lambda: a # hoisted allocation (a pipeline block, an owned call) that flushed to # the enclosing function's statement list would reference lambda params # from outside the lambda struct (undeclared identifier in Zig). - return_value, return_pending = lower_head { lower(return_expr) } - body_mir.concat(hoist_unhoisted_return_allocs( - [*return_pending, MIR::ReturnStmt.new(return_value)], - [lambda_return], - )) + # The tail value leaves the lambda's frame, so it is built on the heap -- + # the placement a written RETURN gets from escape analysis, which never + # sees this synthesized one. A pipeline is the exception: escape analysis + # is the single writer of ITS placement (INV-16), and a pipeline whose + # placement it left on the frame still fails the checker here rather than + # being silently rebuilt somewhere the accumulator does not follow. + tail_alloc = lambda_tail_pipeline?(return_expr) ? nil : :heap + return_value, return_pending = runtime_state.with_rt_name("_rt") do + lower_head { tail_alloc ? with_decl_alloc(tail_alloc) { lower(return_expr) } : lower(return_expr) } + end + capture_state.current_lambda_pointer_params = previous_pointer_params + body_mir.concat([*return_pending, MIR::ReturnStmt.new(return_value)]) # Lambda bodies are nested functions, not ordinary expression children of # the enclosing routine. Run the same allocation normalization and # ownership finalization that a top-level function body receives so an # owned/fallible final expression is hoisted inside the lambda rather than # leaking an unhoisted BlockExpr or TryExpr into its ReturnStmt. + # + # Finalization is what stamps the ownership facts that make a value block + # read as owned, so the return hoist has to run AFTER it -- before, the + # block still looks non-allocating and the hoist skips it. The hoisted + # binding then needs its own finalization pass. body_mir = append_ownership_transfers_for_mir_body(body_mir) + hoisted_returns = hoist_unhoisted_return_allocs(body_mir, [lambda_return]) + unless hoisted_returns.length == body_mir.length + body_mir = append_ownership_transfers_for_mir_body(hoisted_returns) + end fn_def = MIR::FnDef.new(fn_name, params_mir, ret_str, body_mir, nil, false, nil) captures = node.captures&.map { |c| diff --git a/compiler/ruby/mir/lowering/literals.rb b/compiler/ruby/mir/lowering/literals.rb index 10ab988bc..c2859583e 100644 --- a/compiler/ruby/mir/lowering/literals.rb +++ b/compiler/ruby/mir/lowering/literals.rb @@ -395,19 +395,36 @@ def hash_literal_empty_needs_alloc?(zig_type) sig { params(node: AST::HashLit, plan: HashLiteralPlan, capability: HashLiteralCapabilityPlan).returns(MIR::BlockExpr) } def non_empty_hash_literal(node, plan, capability) T.bind(self, MIRLowering) rescue nil - items = T.let([], T::Array[MIR::Stmt]) + items = T.let([], T::Array[MIR::Emittable]) + # Pairs now nest their own literals inside this block, so the label must be + # unique per literal or an inner map collides with its enclosing one. + literal_id = lowering_counters.next_block_expr_id + label = "__hm_blk_#{literal_id}" + hm_name = "__hm_#{literal_id}" alloc_expr = hash_literal_allocator_expr(plan) - items << MIR::Let.new("__hm", capability.init_value || hash_literal_init_struct(capability.zig_type, plan.alloc, true), true, nil, nil) + items << MIR::Let.new(hm_name, capability.init_value || hash_literal_init_struct(capability.zig_type, plan.alloc, true), true, nil, nil) node.pairs.each do |key_node, val_node| - items << hash_literal_put_stmt(key_node, val_node, plan, alloc_expr) + # Each pair's hoisted temps belong to the pair, not to the enclosing + # statement. Draining them at statement level leaves every pair's + # errdefer pending for the rest of the literal, and Zig re-emits the + # whole pending set at every `try` -- quadratic machine code in the + # number of entries. The put consumes the temps in the same scope, so + # confining them to a per-pair block keeps the pending set constant. + outer_pending = function_state.pending_stmts + function_state.pending_stmts = [] + put_stmt = hash_literal_put_stmt(key_node, val_node, plan, alloc_expr, hm_name) + pair_stmts = function_state.pending_stmts + function_state.pending_stmts = outer_pending + items << (pair_stmts.empty? ? put_stmt : MIR::BlockExpr.new(nil, pair_stmts + [put_stmt])) end - result = hash_literal_result(MIR::Ident.new("__hm"), plan, capability) + result = hash_literal_result(MIR::Ident.new(hm_name), plan, capability) if capability.wraps_result? - items << MIR::Let.new("__hm_wrapped", result, false, Type.new(plan.type_info), nil) - result = MIR::Ident.new("__hm_wrapped") + wrapped_name = "__hm_wrapped_#{literal_id}" + items << MIR::Let.new(wrapped_name, result, false, Type.new(plan.type_info), nil) + result = MIR::Ident.new(wrapped_name) end - items << MIR::BreakStmt.new("__hm_blk", result) - block = MIR::BlockExpr.new("__hm_blk", items) + items << MIR::BreakStmt.new(label, result) + block = MIR::BlockExpr.new(label, items) block.result_type = Type.new(plan.type_info) block end @@ -424,12 +441,21 @@ def hash_literal_allocator_expr(plan) ) end - sig { params(key_node: AST::Node, val_node: AST::Node, plan: HashLiteralPlan, alloc_expr: MIR::MethodCall).returns(MIR::ExprStmt) } - def hash_literal_put_stmt(key_node, val_node, plan, alloc_expr) + sig { params(key_node: AST::Node, val_node: AST::Node, plan: HashLiteralPlan, alloc_expr: MIR::MethodCall, hm_name: String).returns(MIR::ExprStmt) } + def hash_literal_put_stmt(key_node, val_node, plan, alloc_expr, hm_name) T.bind(self, MIRLowering) rescue nil key_mir = lower(key_node) + value_type = plan.type_info.value_type raw_value = with_decl_alloc(plan.alloc) do - materialize_owned_sink_value(lower(val_node), val_node, plan.alloc) + # The map owns its values, so a value must live in the map's allocator -- + # storing a @rodata literal directly means the map's cleanup frees + # read-only memory. This is the placement step the list literal does. + # A nested aggregate value builds against the map's VALUE type; without + # it the inner literal guesses from its own items and a `{K}{K}V` map + # stores the inner map's entries directly in the outer one. + lowered = value_type ? with_expected_type(value_type) { lower(val_node) } : lower(val_node) + placed = value_type ? place_value_for_destination(lowered, val_node, plan.alloc, value_type) : lowered + materialize_owned_sink_value(placed, val_node, plan.alloc, value_type) end value_mir = hoist_alloc(raw_value, val_node, err_cleanup: true) operands = ownership_operands_for_value(key_mir, key_node, "hash literal key", plan.alloc) + @@ -441,7 +467,7 @@ def hash_literal_put_stmt(key_node, val_node, plan, alloc_expr) 4, ) MIR::ExprStmt.new( - MIR::MethodCall.new(MIR::Ident.new("__hm"), "put", [alloc_expr, alloc_expr, key_mir, value_mir], true, put_contract), + MIR::MethodCall.new(MIR::Ident.new(hm_name), "put", [alloc_expr, alloc_expr, key_mir, value_mir], true, put_contract), false, ) end @@ -487,6 +513,10 @@ def list_literal_plan(node) def hash_literal_plan(node) T.bind(self, MIRLowering) rescue nil expected_ti = Type.from_node(function_state.current_expected_type) + # A map literal filling an OPTIONAL slot still builds a map: taking the + # expected type as-is renders the container as `?CheatLib.StringMap(V)`, + # which is not a struct literal Zig can initialize. + expected_ti = expected_ti.non_optional_type if expected_ti&.optional? ti = if expected_ti&.map? expected_ti else diff --git a/compiler/ruby/mir/lowering/state.rb b/compiler/ruby/mir/lowering/state.rb index c3ffecbba..9d2075695 100644 --- a/compiler/ruby/mir/lowering/state.rb +++ b/compiler/ruby/mir/lowering/state.rb @@ -76,6 +76,9 @@ class ConstInitEntry < T::Struct const :zig_type, String const :init, MIR::Node const :type_info, T.untyped, default: nil + # Statements the initializer hoisted (owned temps and their marks). They must + # run inside the init prologue, ahead of the value that references them. + const :prelude, T::Array[MIR::Node], default: [] end class MIRLoweringProgramState < T::Struct @@ -102,6 +105,13 @@ class MIRLoweringProgramState < T::Struct prop :fn_nodes, FnNodeMap, factory: -> { {} } prop :function_counter_snapshots, T::Hash[String, MIRLoweringCounterSnapshot], factory: -> { {} } prop :runtime_init_consts, T::Array[ConstInitEntry], factory: -> { [] } + # Zig aliases of imported modules that declare runtime-initialized consts, + # in require order -- the root calls each before its own initializers. + prop :module_const_inits, T::Set[String], factory: -> { Set.new } + # Names declared by a module-level `MUTABLE x = ...`. Their storage is the + # program lifetime, so an assignment into one targets the heap allocator and + # drops nothing. + prop :module_global_names, T::Set[String], factory: -> { Set.new } end class MIRLoweringCaptureState < T::Struct @@ -111,6 +121,9 @@ class MIRLoweringCaptureState < T::Struct CaptureSymbols = T.type_alias { T::Hash[String, SymbolEntry] } prop :current_bg_pointer_captures, T.nilable(T::Set[String]), default: nil + # MUTABLE lambda parameters arrive as pointers, so `&p` inside the body must + # not take a second address. + prop :current_lambda_pointer_params, T::Set[String], factory: -> { Set.new } prop :current_fiber_capture_symbols, CaptureSymbols, factory: -> { {} } prop :do_capture_map, T.nilable(CaptureMap), default: nil prop :current_stream_is_inf, T.nilable(T::Boolean), default: nil diff --git a/compiler/ruby/mir/lowering/variables.rb b/compiler/ruby/mir/lowering/variables.rb index 048fe63b6..a85d3329f 100644 --- a/compiler/ruby/mir/lowering/variables.rb +++ b/compiler/ruby/mir/lowering/variables.rb @@ -149,6 +149,7 @@ def lower_var_decl(node) facts = var_decl_facts(node) return lower_module_const(node, facts) if node.module_const + return lower_module_global(node, facts) if program_state.module_global_names.include?(node.name.to_s) # Every allocating sub-expression of the value -- collection init, # pipeline, COLLECT, toList, concat -- inherits this binding's @@ -240,10 +241,15 @@ def lower_module_const(node, facts) # order, with a real runtime at the top of clearMain (see # inject_const_init!). The value lives in the program arena and every use # borrows it - it is never moved or freed per scope. - heap_value = with_decl_alloc(:heap) { lower(node.value) } - if facts.has_mir_drop || mir_allocates?(heap_value) - return lower_runtime_init_const(node, facts, safe_name, heap_value) + heap_value, const_pending = lower_head { with_decl_alloc(:heap) { lower(node.value) } } + # An initializer whose parts were already hoisted looks non-allocating by + # the time we see it -- the allocation moved into the pending temps, which + # have nowhere to live at container scope. It needs the same runtime-init + # prologue an obviously-allocating initializer takes. + if facts.has_mir_drop || mir_allocates?(heap_value) || const_pending.any? + return lower_runtime_init_const(node, facts, safe_name, heap_value, const_pending) end + function_state.pending_stmts.concat(const_pending) init = with_decl_alloc(facts.decl_alloc) do lower_var_decl_init(node, facts.ft, facts.bare_zig, facts.has_caps, facts.decl_alloc) @@ -261,21 +267,42 @@ def lower_module_const(node, facts) let end + # A module-level MUTABLE global has the same container-scope problem a CONST + # does: its initializer's hoisted temps have nowhere to live. Route an + # initializer that allocates (or hoists) through the same init prologue; the + # storage stays a `var` because the binding is mutable. + sig { params(node: AST::VarDecl, facts: VarDeclFacts).returns(MIR::NodeRoot) } + def lower_module_global(node, facts) + T.bind(self, MIRLowering) rescue nil + safe_name = var_decl_safe_name(node, false) + function_state.binding_types[safe_name] = facts.ft + heap_value, pending = lower_head { with_decl_alloc(:heap) { lower(node.value) } } + if facts.has_mir_drop || mir_allocates?(heap_value) || pending.any? + return lower_runtime_init_const(node, facts, safe_name, heap_value, pending) + end + + function_state.pending_stmts.concat(pending) + MIR::Let.new(safe_name, heap_value, true, facts.annotation, nil) + end + # Emit the storage node for a runtime-initialized CONST and record its # heap-allocated initializer for clearMain's ordered init prologue. The value # transfers into the program-lifetime global (owned sink) with no per-scope # cleanup; a fallible (RAISE-able) initializer is rejected - a CONST has no # error channel (allocation FAULTs like OOM are still permitted). - sig { params(node: AST::VarDecl, facts: VarDeclFacts, safe_name: String, value: MIR::Node).returns(MIR::NodeRoot) } - def lower_runtime_init_const(node, facts, safe_name, value) + sig { params(node: AST::VarDecl, facts: VarDeclFacts, safe_name: String, value: MIR::Node, prelude: T::Array[MIR::Node]).returns(MIR::NodeRoot) } + def lower_runtime_init_const(node, facts, safe_name, value, prelude = []) T.bind(self, MIRLowering) rescue nil annotation = facts.annotation - zig_type = annotation ? annotation.nested_zig_type : transpile_type(facts.ft.resolved.to_s) + # `facts.ft.resolved.to_s` is the legacy CLEAR spelling ("String[SET]"), + # which transpile_type passes straight through. The Type renders itself. + zig_type = annotation ? annotation.nested_zig_type : facts.ft.nested_zig_type program_state.runtime_init_consts << ConstInitEntry.new( name: safe_name, zig_type: zig_type, init: value, - type_info: facts.ft + type_info: facts.ft, + prelude: prelude ) MIR::ModuleVar.new(safe_name, zig_type, node.const_visibility) end @@ -392,7 +419,10 @@ def var_decl_facts(node) # what makes "@::" combinations compose without # per-shape × per-cap glue. has_caps = !!((ft.any_sync? || ft.ownership != :affine) && !ft.striped?) - bare_ft = has_caps ? ft.bare_data_type : ft + # `?T@multiowned` is an OPTIONAL HANDLE (`?Rc(T)`), not a handle to an + # optional: the carrier wraps the payload and the optional wraps the + # carrier, so the wrap is spelled against the non-optional payload. + bare_ft = has_caps ? ft.non_optional_type.bare_data_type : ft bare_zig = transpile_type(bare_ft) VarDeclFacts.new( @@ -476,7 +506,9 @@ def var_decl_safe_name(node, has_mir_drop) # the names unique so the checker sees independent containers. original_safe = safe_name if alloc_marked_names.key?(safe_name) - safe_name = "#{safe_name}_L#{function_relative_line(node.line)}" + # Suffix the CLEAR name, not the escaped spelling: `@"type"_L2` splices + # the escape into the middle of a new identifier. + safe_name = zig_safe_name("#{node.name}_L#{function_relative_line(node.line)}") end alloc_marked_names[safe_name] = true decl_name_map[node.object_id] = safe_name @@ -625,9 +657,18 @@ def allocating_init_var_decl_plan(node, facts, safe_name, init, let_node) T.bind(self, MIRLowering) rescue nil mir_alloc = mir_owned_alloc(init) || facts.decl_alloc alloc_mark = var_decl_alloc_mark(safe_name, mir_alloc, facts.ft, facts.binding_entry) + # An AllocMark asserts the binding owns an allocation. A rodata binding -- + # an interned symbol, a static string slice -- owns nothing whatever its + # initializer allocated along the way, and marking it owned leaves a + # scope-local the checker can never see released. + return MIR::MaterializationPacket.value_only(let_node) if facts.ft.rodata? return MIR::MaterializationPacket.owned(alloc_mark, let_node) unless type_requires_alloc_cleanup?(facts.ft, mir_alloc) - cleanup_entry = T.must(hoist_cleanup_entry(init, node)) + # The AllocMark above already fixed this binding's allocator. A cleanup + # recipe inherited from the init expression may name a different one + # (a heap String recipe for a frame-placed element view), and a binding + # has exactly one allocator (INV-1). + cleanup_entry = T.must(hoist_cleanup_entry(init, node)).with_alloc(mir_alloc) build_drop_entry!(cleanup_entry, node.full_type!, node) mark_guarded_cleanup_name!(safe_name) if cleanup_entry.has_moved_guard? MIR::MaterializationPacket.owned(alloc_mark, let_node, MIR::Cleanup.new(safe_name, cleanup_entry)) @@ -733,19 +774,43 @@ def lower_var_decl_init(node, ft, bare_zig, has_caps, decl_alloc) end retain_source = var_decl_retain_source(node.value) - if retain_source.is_a?(AST::Identifier) && node.value.was_moved != true && rc_retain_needed?(retain_source) - return make_rc_retain(retain_source) + # rc_retain_needed? is false for anything but an Identifier; the is_a? + # check only surfaces that fact for the type checker. + return make_rc_retain(retain_source) if retain_source.is_a?(AST::Identifier) && node.value.was_moved != true && rc_retain_needed?(retain_source) + + # A declaration that wraps its value in a carrier receives the PAYLOAD, not + # the carrier: placing against the carrier type coerces a plain value to + # Rc(T)/Arc(T) and the wrap then builds a handle out of that lie. + wraps_value = has_caps && !source_already_has_declared_capability?(node.value, ft) + value_ft = wraps_value ? ft.non_optional_type.bare_data_type : ft + placed = with_expected_type(value_ft) { lower(node.value) } + placed = place_value_for_destination(placed, node.value, decl_alloc, value_ft) + # The handle OWNS its payload. Wrapping a borrowed view (a union match + # payload, a field read) would hand the handle storage someone else frees. + if wraps_value && borrowed_wrap_source?(node.value) + placed = MIR::DeepCopy.new(placed, transpile_type(value_ft), nil, :full_value, decl_alloc) end - - placed = with_expected_type(ft) { lower(node.value) } - placed = place_value_for_destination(placed, node.value, decl_alloc, ft) - if has_caps && !capability_wrapped_mir?(placed) && !source_already_has_declared_capability?(node.value, ft) + if wraps_value && !capability_wrapped_mir?(placed) compose_capability_wrap(placed, bare_zig, ft, decl_alloc) else placed end end + # Is this value a borrowed view rather than an owned value? A borrowed + # HANDLE is excluded: an Rc/Arc is retained, never structurally duplicated. + sig { params(value: T.nilable(AST::Node)).returns(T::Boolean) } + def borrowed_wrap_source?(value) + return false unless value.is_a?(AST::Locatable) + + source = value.full_type!(context: "carrier wrap source") + return false if source.any_rc? + + source.borrowed_reference? + rescue StandardError + false + end + sig { params(value: AST::Node).returns(AST::Node) } def var_decl_retain_source(value) return value.value if value.is_a?(AST::MoveNode) @@ -903,7 +968,10 @@ def lower_bind_expr(node) # binding's allocator (one allocator per binding). binding_entry = cleanup_entry_for_ast_binding(node) || function_state.bindings[node.name.to_s] || CleanupEntry::NONE heap_return_var = current_function_heap_carry_return_var?(node.name.to_s) - assign_alloc = if heap_return_var + module_global = program_state.module_global_names.include?(node.name.to_s) + assign_alloc = if heap_return_var || module_global + # A module global lives for the whole program: its storage is the heap + # and no scope drops it. :heap else rp ? alloc_from_sym(rp.alloc!) : (binding_entry.present? ? binding_entry.alloc : nil) @@ -959,9 +1027,31 @@ def lower_destructuring_assignment(node) T.bind(self, MIRLowering) rescue nil value = T.cast(lower(node.value), MIR::Emittable) targets = node.targets.map { |target| lower_destructure_target(target) } + # A target DECLARED here owns nothing of its own -- the temp the aggregate + # arrived in stays its owner. A target that already exists brings its own + # cleanup, so the temp has to release or both free the same pieces. + function_state.pending_stmts.concat(destructure_source_transfer(node, value)) MIR::DestructureSet.new(targets, value) end + sig { params(node: AST::DestructuringAssignment, value: MIR::Emittable).returns(T::Array[MIR::Node]) } + def destructure_source_transfer(node, value) + T.bind(self, MIRLowering) rescue nil + name = value.is_a?(MIR::Ident) ? value.name.to_s : nil + return [] unless name + return [] unless function_state.guarded_cleanup_names[name] + + reassigns_owner = node.targets.any? do |target| + next false if target.name.to_s == "_" + next false if target.symbol&.reg.equal?(target) + function_state.guarded_cleanup_names[zig_safe_name(target.name)] || + function_state.bindings[target.name.to_s]&.needs_cleanup? + end + return [] unless reassigns_owner + + ownership_transfer_marks(name, :block_result, move_guarded: true) + end + sig { params(target: AST::DestructureTarget).returns(MIR::DestructureTarget) } def lower_destructure_target(target) T.bind(self, MIRLowering) diff --git a/compiler/ruby/mir/mir.rb b/compiler/ruby/mir/mir.rb index 50d9b5373..5315a0ad4 100644 --- a/compiler/ruby/mir/mir.rb +++ b/compiler/ruby/mir/mir.rb @@ -407,6 +407,17 @@ def stmt?; false; end def expr?; false; end sig { returns(OwnershipEffect) } def ownership_effect; OwnershipEffect.none; end + # Does this node MATERIALIZE a value, as opposed to projecting one out of + # something that already exists? A construction owns what it yields; a + # field read, an element read, an unwrap or a cast is a view of storage + # someone else owns, and giving one of those a cleanup frees storage its + # owner still holds. + # + # It lives here rather than in a list somewhere because it is a fact about + # the node. A list has to be found and updated when a construction is + # added; an override next to its siblings does not. + sig { returns(T::Boolean) } + def materializes_value?; false; end sig { returns(T::Array[Emittable]) } def child_exprs; EMPTY_CHILD_EXPRS; end sig { returns(T::Array[Emittable]) } @@ -3046,6 +3057,8 @@ def body_slots HeapCreate = Struct.new(:zig_type, :init, :alloc, :label) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { params(zig_type: String, init: T.untyped, alloc: Symbol, label: T.nilable(String)).void } def initialize(zig_type, init, alloc, label = nil) super(zig_type, init, alloc, label) @@ -3068,6 +3081,8 @@ def ownership_effect DupeSlice = Struct.new(:source, :alloc) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { params(source: T.untyped, alloc: Symbol).void } def initialize(source, alloc) super(source, alloc) @@ -3088,6 +3103,8 @@ def ownership_effect AllocSlice = Struct.new(:elem_type, :len, :alloc) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { params(elem_type: String, len: T.untyped, alloc: Symbol).void } def initialize(elem_type, len, alloc) super(elem_type, len, alloc) @@ -3192,6 +3209,8 @@ def child_exprs = compact_child_exprs([ptr]) :alloc, :copy_shape, :type_info) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig do params( source: T.untyped, @@ -3241,6 +3260,8 @@ def ownership_effect :capacity) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { params(zig_type: String, strategy: Symbol, alloc: T.nilable(Symbol), capacity: T.untyped).void } def initialize(zig_type, strategy, alloc, capacity) super(zig_type, strategy, alloc, capacity) @@ -3269,6 +3290,9 @@ def ownership_effect :own_fn, # "arcCreate", "rcCreate", nil :alloc) do extend T::Sig + sig { returns(T::Boolean) } + def materializes_value? = true + include Expr sig { params(inner: T.untyped, zig_base: String, strategy: Symbol, sync_fn: T.nilable(String), sync_type: T.nilable(String), own_fn: T.nilable(String), alloc: Symbol).void } def initialize(inner, zig_base, strategy, sync_fn, sync_type, own_fn, alloc) @@ -3417,6 +3441,8 @@ def ownership_effect MakeList = Struct.new(:elem_type, :items, :alloc, :minimum_capacity) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { params(elem_type: String, items: T::Array[Emittable], alloc: Symbol, minimum_capacity: T.nilable(Integer)).void } def initialize(elem_type, items, alloc, minimum_capacity = nil) super(elem_type, items, alloc, minimum_capacity) @@ -4247,6 +4273,8 @@ def child_exprs = compact_child_exprs([value]) TupleLiteral = Struct.new(:items) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { returns(T::Array[Emittable]) } def child_exprs values = T.let([], T::Array[Emittable::ChildExprValue]) @@ -4300,6 +4328,8 @@ def child_exprs = compact_child_exprs([left, right]) StructInit = Struct.new(:zig_type, :fields) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true # zig_type: String or nil (nil -> anonymous .{}) # fields: [MIR::StructInitField] (legacy hash fields are still readable) sig { returns(T::Array[Emittable]) } @@ -4325,6 +4355,8 @@ def ownership_effect ArrayInit = Struct.new(:elem_type, :count, :items) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { returns(T::Array[Emittable]) } def child_exprs = compact_child_exprs([items]) sig { returns(T::Array[Emittable]) } @@ -4429,6 +4461,8 @@ def ownership_effect ConcatStr = Struct.new(:parts, :alloc, :rt_expr) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { params(parts: T::Array[T.untyped], alloc: Symbol, rt_expr: T.nilable(String)).void } def initialize(parts, alloc, rt_expr) super(parts, alloc, rt_expr) @@ -4918,6 +4952,8 @@ def expr OwnedSlice = Struct.new(:expr, :alloc) do extend T::Sig include Expr + sig { returns(T::Boolean) } + def materializes_value? = true sig { params(expr: Emittable, alloc: Symbol).void } def initialize(expr, alloc) super(expr, alloc) diff --git a/compiler/ruby/mir/mir_checker.rb b/compiler/ruby/mir/mir_checker.rb index 547869103..7ff2b50dc 100644 --- a/compiler/ruby/mir/mir_checker.rb +++ b/compiler/ruby/mir/mir_checker.rb @@ -962,6 +962,11 @@ def normalize_guarded_conditional_releases!(states) sig { params(expected: LinearOwnershipState, actual: LinearOwnershipState, label: String).void } def linear_require_same_state!(expected, actual, label) + # A `_moved`-guarded binding released on only one path through the body is + # exactly what the guard exists for -- the same normalization a branch join + # gets. Without it, a loop that reassigns a guarded handle and returns early + # on some iterations reads as an unprovable state change. + normalize_guarded_conditional_releases!([expected, actual]) return if expected.same_state?(actual) @errors << error(:OWNERSHIP_UNVERIFIED_PATH, label, diff --git a/compiler/ruby/mir/mir_lowering.rb b/compiler/ruby/mir/mir_lowering.rb index 06fa07ab7..a483761d4 100644 --- a/compiler/ruby/mir/mir_lowering.rb +++ b/compiler/ruby/mir/mir_lowering.rb @@ -56,7 +56,13 @@ class MIRLowering # backend namespace so ordinary CLEAR parameters such as `path` remain legal. sig { params(name: String).returns(String) } def zig_module_alias(name) - "__clear_module_#{name.gsub('.', '_')}" + # Only the OWNING package of a multi-file group is built, so every + # reference -- the import, the type aliases, and a cross-package call's + # qualifier -- has to name the owner. Otherwise the same CLEAR type + # reaches Zig as two distinct types. + importer = program_state.importer + canonical = importer && importer.respond_to?(:owning_package_name) ? importer.owning_package_name(name) : name + "__clear_module_#{canonical.gsub('.', '_')}" end OwnershipFact = T.type_alias do @@ -642,7 +648,14 @@ def next_stream_literal_id sig { params(mir: MIR::Node, ast_node: AST::Node, dest_alloc: T.nilable(Symbol), dest_type: T.nilable(Type::TypeInput)).returns(MIR::Node) } def place_value_for_destination(mir, ast_node, dest_alloc, dest_type = nil) plan = destination_placement_plan(mir, ast_node, dest_alloc, dest_type) - plan.place(self, mir, ast_node) + placed = plan.place(self, mir, ast_node) + # A symbol reaching a String destination widens to its bytes. Placement + # decides HOW the value is stored; this decides WHAT is stored, and only + # the source type can answer it. + dst = dest_type.is_a?(Type) ? dest_type : (dest_type ? Type.new(dest_type) : nil) + return placed unless dst&.byte_string? + + widen_symbol_to_bytes(placed, ast_node) end sig { params(value: MIR::Node, shape: AsyncResultShape).returns(MIR::Node) } @@ -1053,12 +1066,29 @@ def place_owned_alloc_mismatch_for_destination(mir, ti, dest_alloc, source_alloc out end + # Duplicating an Rc/Arc handle is a RETAIN, never a structural copy -- and an + # OPTIONAL handle retains inside the `if present` arm. Returns nil when the + # destination is not a handle. + sig { params(mir: MIR::Node, ti: Type).returns(T.nilable(MIR::Node)) } + def retain_handle_for_destination(mir, ti) + payload = ti.optional? ? ti.wrapped_type : ti + return nil unless payload&.any_rc? + + fn = payload.shared? ? "arcRetain" : "rcRetain" + zig = rc_payload_zig_type(payload) + return MIR::RcRetain.new(mir, zig, fn) unless ti.optional? + + capture = "__retain_rc_#{lowering_counters.next_tmp_id}" + retained = MIR::IfOptional.new(mir, capture, MIR::RcRetain.new(MIR::Ident.new(capture), zig, fn), MIR::Lit.new("null")) + retained.result_type = Type.new(ti) + retained + end + sig { params(mir: MIR::Node, ti: Type, dest_alloc: Symbol).returns(MIR::Node) } def copy_owned_value_for_destination(mir, ti, dest_alloc) return MIR::DupeSlice.new(mir, dest_alloc) if ti.string? - if ti.any_rc? - return MIR::RcRetain.new(mir, rc_payload_zig_type(ti), ti.shared? ? "arcRetain" : "rcRetain") - end + retained = retain_handle_for_destination(mir, ti) + return retained if retained MIR::DeepCopy.new( mir, @@ -1096,6 +1126,20 @@ def place_string_or_for_heap_destination(mir, ast_node) end end + # Widen a `String@symbol` to the bytes behind it. A Symbol is an interned + # handle with its own Zig type, so copying one into an owned String has to + # read `.bytes` first -- this is the borrow that widening always was, made + # explicit now that the two types are distinct. + sig { params(mir: MIR::Node, source_node: T.nilable(AST::Node)).returns(MIR::Node) } + def widen_symbol_to_bytes(mir, source_node) + return mir unless source_node + + ti = Type.from_node!(source_node, context: "symbol widening") + return mir unless ti.symbol? + + MIR::FieldGet.new(mir, "bytes") + end + sig { params(mir: MIR::Node, dst_ti: Type, dest_alloc: Symbol).returns(MIR::Node) } def place_owned_branch_value_for_destination(mir, dst_ti, dest_alloc) # Nested optional merges have not yet received finalized ownership facts @@ -1112,9 +1156,8 @@ def place_owned_branch_value_for_destination(mir, dst_ti, dest_alloc) end return place_owned_alloc_mismatch_for_destination(mir, dst_ti, dest_alloc, owned_alloc) if owned_alloc return MIR::DupeSlice.new(mir, dest_alloc) if dst_ti.string? - if dst_ti.any_rc? - return MIR::RcRetain.new(mir, rc_payload_zig_type(dst_ti), dst_ti.shared? ? "arcRetain" : "rcRetain") - end + retained = retain_handle_for_destination(mir, dst_ti) + return retained if retained # Lazy branch blocks cannot be hoisted outside their branch, but a # recursive owned result still needs a named source owner when it is @@ -1133,6 +1176,53 @@ def place_owned_branch_value_for_destination(mir, dst_ti, dest_alloc) ) end + # A value block that ends in `binding?` hands out the payload but keeps its + # own guarded cleanup. Only a consumer that TAKES the result can know the + # transfer is due, so claim it here rather than in the block. + # Returns whether the transfer was claimed. A claim MAKES the destination the + # owner, whatever the result expression looks like -- `slot?` reads as a view + # of the block's binding, but once the block hands its claim over, the value + # is the consumer's to drop. + sig { params(mir: MIR::Node).returns(T::Boolean) } + def claim_block_result_ownership!(mir) + return false unless mir.is_a?(MIR::BlockExpr) + return true if mir.body.any? { |stmt| stmt.is_a?(MIR::TransferMark) && stmt.target == :block_result } + + break_index = mir.body.rindex { |stmt| stmt.is_a?(MIR::BreakStmt) } + return false unless break_index + + owner = mir_ident_names(T.cast(mir.body[break_index], MIR::BreakStmt).value).first + return false unless owner + + cleanup = mir.body.find do |stmt| + stmt.is_a?(MIR::Cleanup) && stmt.name.to_s == owner && stmt.cleanup_entry&.[](:has_moved_guard) + end + return false unless cleanup + + mir.body[break_index, 0] = ownership_transfer_marks(owner, :block_result, move_guarded: true) + true + end + + # Does the materialized source own what it yields, or is it a view of storage + # that outlives it? This path is reached for anything that must be named + # before it is copied, which `mir_allocates?` answers for the whole subtree -- + # a lookup keyed by an allocating call "allocates" while still handing back a + # view, and cleaning that up frees storage the container still holds. + sig { params(mir: MIR::Node).returns(T::Boolean) } + def owned_branch_source_owns?(mir) + result = mir + if mir.is_a?(MIR::BlockExpr) + result = T.cast(mir.body.reverse.find { |stmt| stmt.is_a?(MIR::BreakStmt) }, T.nilable(MIR::BreakStmt))&.value + return true unless result + end + # An identifier's ownership is a fact about its BINDING, recorded when the + # binding was lowered; this predicate is not the authority on it. + return true if result.is_a?(MIR::Ident) + return true if MIR::OwnershipEffect.of(result).produces_owned + + result.materializes_value? + end + sig { params(mir: MIR::Node, type_info: Type, dest_alloc: Symbol).returns(MIR::BlockExpr) } def copy_lazy_owned_branch_for_destination(mir, type_info, dest_alloc) tmp_id = lowering_counters.next_tmp_id @@ -1140,10 +1230,24 @@ def copy_lazy_owned_branch_for_destination(mir, type_info, dest_alloc) source_name = "__owned_branch_src_#{tmp_id}" copy_name = "__owned_branch_copy_val_#{tmp_id}" + # This path TAKES the block's result (the source binding below owns it and + # frees it). A block that yields `binding?` never released its own claim, + # so ask for the transfer here -- the block cannot know whether its + # consumer takes or merely borrows. + claimed = claim_block_result_ownership!(mir) source_alloc = mir_owned_alloc(mir) || MIR::OwnershipEffect.alloc_of(mir) || dest_alloc - source_cleanup = CleanupEntry.build(:uniform, alloc: source_alloc, has_moved_guard: false, - zig_type: type_info.zig_type) - build_drop_entry!(source_cleanup, type_info, nil) + # ... but only when the block's result is genuinely owned. A block whose + # result is a BORROW -- a container lookup whose key needed a temp, so the + # lookup got wrapped in a block -- owns nothing, and cleaning it up frees + # storage still held by the container. The copy below is what the + # destination keeps either way. + source_owned = claimed || owned_branch_source_owns?(mir) + source_cleanup = nil + if source_owned + source_cleanup = CleanupEntry.build(:uniform, alloc: source_alloc, has_moved_guard: false, + zig_type: type_info.zig_type) + build_drop_entry!(source_cleanup, type_info, nil) + end source = MIR::BindingMaterialization.new( name: source_name, expr: mir, @@ -1151,9 +1255,10 @@ def copy_lazy_owned_branch_for_destination(mir, type_info, dest_alloc) type_info: type_info, mutable: false, cleanup_entry: source_cleanup, + ownership_tracked: source_owned, ) - copied_expr = MIR::DeepCopy.new( + copied_expr = retain_handle_for_destination(MIR::Ident.new(source_name), type_info) || MIR::DeepCopy.new( MIR::Ident.new(source_name), type_info.zig_type, nil, @@ -1477,6 +1582,7 @@ def apply_lowered_coercion(mir, node) # Optionality is encoded inside NodeRef's zero sentinel; T@node and # ?T@node therefore have the same Zig representation and need no cast. return mir if coerced_type.node_reference? && actual_type.node_reference? + return mir if carrier_only_coercion?(actual_type, coerced_type) if coerced_type.node_reference? && !actual_type.node_reference? if actual_type.resolved == :NIL @@ -1491,6 +1597,19 @@ def apply_lowered_coercion(mir, node) mir_cast(mir, actual_type, coerced_type) || mir end + # A coercion that only ADDS an ownership/sync carrier is performed + # structurally by the declaration's CapWrap. A Zig cast for it would claim a + # plain value already is an Rc/Arc handle. + sig { params(actual_type: Type, coerced_type: Type).returns(T::Boolean) } + def carrier_only_coercion?(actual_type, coerced_type) + coerced_payload = coerced_type.non_optional_type + actual_payload = actual_type.non_optional_type + return false unless coerced_payload.any_rc? || coerced_payload.any_sync? + return false if actual_payload.any_rc? || actual_payload.any_sync? + + coerced_payload.resolved == actual_payload.resolved + end + sig { params(mir: MIR::Emittable, node: AST::Locatable, actual_type: Type, coerced_type: Type).returns(T.nilable(MIR::Emittable)) } def lower_union_payload_coercion(mir, node, actual_type, coerced_type) target_type = coerced_type.value_payload_type @@ -1753,6 +1872,7 @@ def append_ownership_finalized_node!(state, node, body, line, col) finalize_nested_mir_bodies!(node, state) stamp_source_line!(node, line, col) append_transfer_marks_to_body!(state, pre_terminator_transfer_marks(node, state.out, body), line, col) + node_index = state.out.length state.out << node append_move_guard_for_transfer_mark!(node, state) surface = scan_ownership_surface!( @@ -1764,15 +1884,28 @@ def append_ownership_finalized_node!(state, node, body, line, col) state.out.concat(surface.facts) mark_ownership_finalized_node!(node) mark_ownership_finalized_nodes!(surface.facts) + transfer_index = state.out.length append_transfer_marks_to_body!( state, ownership_transfers_for_targets(surface.transfer_targets, state), line, col, ) + # A MoveMark has to PRECEDE the move it guards. When the consuming node is + # a terminator -- `RETURN Wrapper{ field: owned }` -- appending after it + # both writes the guard too late and emits statements Zig rejects as + # unreachable. + if terminator_stmt?(node) && state.out.length > transfer_index + state.out[node_index, 0] = T.must(state.out.slice!(transfer_index..)) + end nil end + sig { params(node: MIR::Node).returns(T::Boolean) } + def terminator_stmt?(node) + node.is_a?(MIR::ReturnStmt) || node.is_a?(MIR::BreakStmt) || node.is_a?(MIR::ContinueStmt) + end + sig { params(state: OwnershipFinalizationContext, marks: T::Array[MIR::Stmt], line: T.nilable(Integer), col: T.nilable(Integer)).void } def append_transfer_marks_to_body!(state, marks, line, col) marks.each do |mark| @@ -2161,7 +2294,11 @@ def append_block_result_transfer!(node, body, state) return if state.body_transfer_mark_names.include?(name) return unless state.alloc_marks.key?(name) || state.body_alloc_mark_names.include?(name) - ownership_transfer_marks(name, :block_result).each do |mark| + # A binding whose cleanup is `_moved`-guarded must have that flag set when + # the block hands its value out, or the value is both transferred out and + # cleaned up on the way. + guarded = state.guarded_cleanup_names.include?(name) + ownership_transfer_marks(name, :block_result, move_guarded: guarded).each do |mark| state.out << mark record_ownership_finalization_node!(state, mark) end @@ -3088,6 +3225,14 @@ def stamp_source_line!(node, line, column = nil) # Like lower_body, but the last user-visible statement becomes break :label expr # instead of a regular statement. Used for IF/MATCH expression branches. + # A NoReturn tail expression (`DEFAULT -> panic("...")`) has no value. + sig { params(node: T.untyped).returns(T::Boolean) } + def noreturn_result_expr?(node) + resolved = node.respond_to?(:resolved_type) ? T.unsafe(node).resolved_type : nil + resolved = resolved.resolved if resolved.is_a?(Type) + resolved == :NoReturn + end + sig { params(stmts: T::Array[LowerableStmt], label: String).returns(T::Array[MIR::Emittable]) } def lower_body_with_break(stmts, label) return [] if stmts.empty? @@ -3105,10 +3250,16 @@ def lower_body_with_break(stmts, label) # Draining the ambient list wholesale scooped hoists that belong to the # ENCLOSING expression (an earlier concat part's owned temp) into this # branch's scope — emitted Zig then referenced them outside the branch. - result_mir, pending = lower_head { lower(T.must(stmts[last_user_idx])) } + result_stmt = T.must(stmts[last_user_idx]) + result_mir, pending = lower_head { lower(result_stmt) } suffix_lowered = lower_body(stmts.drop(last_user_idx + 1)) - tail = pending + suffix_lowered + [MIR::BreakStmt.new(label, T.cast(result_mir, MIR::Node))] + # A NoReturn tail yields no value to break with: Zig reads + # `break :blk @panic(...)` as unreachable code. + terminator = noreturn_result_expr?(result_stmt) ? + T.cast(result_mir, MIR::Emittable) : + MIR::BreakStmt.new(label, T.cast(result_mir, MIR::Node)) + tail = pending + suffix_lowered + [terminator] prefix_lowered + normalize_allocating_mir_body(tail) end @@ -3149,7 +3300,7 @@ def lower_program(node, use_c_allocator: false, needs_safety: false, use_debug_a lowering_counters.restore!(seed) if seed program_state.function_counter_snapshots[stmt.name] = lowering_counters.snapshot end - lowered = lower(stmt) + lowered = lower_top_level(stmt) reject_module_scope_cleanup!(stmt, lowered) append_lowered_items!(LoweredItemTarget.new(items: items, line: stmt.token&.line), lowered) end @@ -3167,18 +3318,38 @@ def lower_program(node, use_c_allocator: false, needs_safety: false, use_debug_a # transfers it into its program-lifetime global (owned sink, no per-scope # cleanup), and call that at the very top of clearMain so every container-scope # read borrows an initialized value. - sig { params(items: T::Array[MIR::Node]).void } - def inject_const_init!(items) + CONST_INIT_FN = "__clear_init_consts" + CONST_INIT_GUARD = "__clear_consts_ready" + + sig { params(items: T::Array[MIR::Node], module_scope: T::Boolean).void } + def inject_const_init!(items, module_scope: false) entries = program_state.runtime_init_consts - return if entries.empty? + module_inits = program_state.module_const_inits.to_a + return if entries.empty? && module_inits.empty? raw = T.let([], T::Array[MIR::Node]) + # Two importers of the same package both call its initializer, so the + # second call has to be a no-op or the first build leaks. + if module_scope + items << MIR::Let.new(CONST_INIT_GUARD, MIR::Lit.new("false"), true, Type.new(:Bool), nil) + raw << MIR::IfStmt.new(MIR::Ident.new(CONST_INIT_GUARD), [MIR::ReturnStmt.new(nil)], nil) + raw << MIR::Set.new(MIR::Ident.new(CONST_INIT_GUARD), MIR::Lit.new("true")) + end + # An imported module's consts must exist before this program's own + # initializers -- or its own body -- can read them. + module_inits.each do |alias_name| + raw << MIR::Call.new( + "#{alias_name}.#{CONST_INIT_FN}", [MIR::Ident.new("rt")], true, false, + MIR::CallableContract.no_ownership(1) + ) + end entries.each do |entry| tmp = "__ci_#{entry.name}" # Construct the value into a heap temp, copy it into the program-lifetime # global, then transfer the temp's ownership to the sink (the global now # owns it program-lifetime; no per-scope cleanup). The store must precede # the transfer so the read is not use-after-transfer. + raw.concat(entry.prelude) raw << MIR::AllocMark.new(tmp, :heap, entry.type_info, :heap) raw << MIR::Let.new(tmp, entry.init, false, nil, nil) raw << MIR::Set.new(MIR::Ident.new(entry.name), MIR::Ident.new(tmp)) @@ -3186,11 +3357,13 @@ def inject_const_init!(items) end body = finalize_synthetic_const_init_body!(raw) items << MIR::FnDef.new( - "__clear_init_consts", + CONST_INIT_FN, [MIR::Param.new("rt", "*Runtime", false)], "void", body, - :private, + # The root calls an imported module's initializer across the Zig module + # boundary, so a module's copy cannot be private. + module_scope ? :pub : :private, true, [] ) @@ -3204,7 +3377,7 @@ def inject_const_init!(items) # in reverse, after every use), then run the ordered init first. entries.reverse_each { |entry| main_fn.body.unshift(MIR::ModuleConstFree.new(entry.name)) } main_fn.body.unshift(MIR::Call.new( - "__clear_init_consts", [MIR::Ident.new("rt")], true, false, MIR::CallableContract.no_ownership(1) + CONST_INIT_FN, [MIR::Ident.new("rt")], true, false, MIR::CallableContract.no_ownership(1) )) end end @@ -3368,36 +3541,72 @@ def lower_module(node) node.statements.each do |stmt| case stmt when AST::FunctionDef - append_lowered_items!(LoweredItemTarget.new(items: fn_items, line: stmt.token.line), lower(stmt)) + append_lowered_items!(LoweredItemTarget.new(items: fn_items, line: stmt.token.line), lower_top_level(stmt)) when AST::StructDef, AST::EnumDef, AST::UnionDef - append_lowered_items!(LoweredItemTarget.new(items: type_items, line: stmt.token.line), lower(stmt)) + append_lowered_items!(LoweredItemTarget.new(items: type_items, line: stmt.token.line), lower_top_level(stmt)) when AST::RequireNode - append_lowered_items!(LoweredItemTarget.new(items: fn_items, line: nil), lower(stmt)) + append_lowered_items!(LoweredItemTarget.new(items: fn_items, line: nil), lower_top_level(stmt)) when AST::ExternFnDecl, AST::ExternStructDecl - append_lowered_items!(LoweredItemTarget.new(items: fn_items, line: stmt.token.line), lower(stmt)) + append_lowered_items!(LoweredItemTarget.new(items: fn_items, line: stmt.token.line), lower_top_level(stmt)) when AST::VarDecl, AST::BindExpr # Module-scope immutable bindings (e.g. frozen membership tables) # become file-scope consts, exactly as lower_program emits them. - lowered = lower(stmt) + lowered = lower_top_level(stmt) reject_module_scope_cleanup!(stmt, lowered) append_lowered_items!(LoweredItemTarget.new(items: fn_items, line: stmt.token.line), lowered) end end + inject_const_init!(fn_items, module_scope: true) LoweredModuleItems.new(items: fn_items, type_items: type_items) end private + # pending_stmts is hoist scratch for the statement being lowered, but + # FunctionState is per-MIRLowering, not per-function. Anything a previous + # top-level statement failed to drain surfaces inside the NEXT function's + # body -- carrying its AllocMark/ErrCleanup groups without the TransferMarks + # that were emitted with the body it actually belongs to. Every top-level + # entry point (program and module) starts a statement with empty scratch. + sig { params(stmt: AST::Node).returns(T.nilable(LoweredMir)) } + def lower_top_level(stmt) + function_state.pending_stmts = [] + # A module-level MUTABLE binding lives for the whole program. Record it + # before lowering so an assignment inside a function knows its destination + # is the heap and that nothing drops it. + program_state.module_global_names.add(stmt.name.to_s) if stmt.is_a?(AST::VarDecl) && !stmt.module_const + lowered = lower(stmt) + # A container-scope declaration cannot carry a statement suffix: Zig reads + # `var x: i64 = 11; _ = &x;` at module scope as a malformed field list. + # The unused-binding suppression only belongs inside a function body. + items = lowered.is_a?(Array) ? lowered : [lowered] + items.each { |item| item.suppression = nil if item.is_a?(MIR::Let) } + lowered + end + # ================================================================ # Name and type helpers # ================================================================ + # Zig identifiers carry no `?`/`!`, but CLEAR (like Ruby) distinguishes + # `raw` from `raw?` and `check` from `check!`. Stripping the mark collapsed + # the pair onto one Zig name -- a duplicate declaration, or worse, a silent + # call to the wrong one. Encode the mark instead. + PREDICATE_SUFFIX = "_p" + BANG_SUFFIX = "_bang" + sig { params(name: String).returns(String) } def zig_safe_name(name) - cleaned = (name.end_with?('!') || name.end_with?('?')) ? name[0..-2] : name + cleaned = if name.end_with?('?') + "#{name[0..-2]}#{PREDICATE_SUFFIX}" + elsif name.end_with?('!') + "#{name[0..-2]}#{BANG_SUFFIX}" + else + name + end cleaned = Compiler::Entrypoint::ZIG_NAME if cleaned == Compiler::Entrypoint::NAME - cleaned = T.must(cleaned) + cleaned = cleaned ZigType.reserved_identifier?(cleaned) ? "@\"#{cleaned}\"" : cleaned end @@ -3577,9 +3786,12 @@ def extract_root_var_name(node) # Produce a MIR::Cast node for type coercion, or nil if no cast needed. # Mirrors transpile_cast logic but returns MIR nodes instead of strings. - sig { params(mir_node: MIR::Node, from_type: Type, to_type: Type::TypeInput).returns(T.nilable(MIR::Cast)) } + sig { params(mir_node: MIR::Node, from_type: Type, to_type: Type::TypeInput).returns(T.nilable(MIR::Node)) } def mir_cast(mir_node, from_type, to_type) + # A NoReturn value (`panic(...)`) coerces to every type in Zig; wrapping it + # in `@as(T, ...)` only produces unreachable code at the use site. from_t = from_type + return mir_node if from_t.resolved == :NoReturn to_t = to_type.is_a?(Type) ? to_type : Type.new(to_type) return nil if from_t.semantic_type_key == to_t.semantic_type_key # @boxed is constructed by destination placement (HeapCreate); it is @@ -3811,7 +4023,7 @@ def lower_struct_lifecycle_methods(node) copy_forbidden = true elsif plan.copy_strategy != :bit_copy clone_fields << name.to_s - field_source = "self.#{name}" + field_source = "self.#{zig_safe_name(name.to_s)}" # The clone temp is a fresh identifier: a field name (always an # identifier) prefixed with `__clone_` is never a Zig keyword, so it # needs no `@"..."` quoting. Quoting it yields invalid Zig (`__clone_@"type"`). @@ -3837,19 +4049,30 @@ def lower_struct_lifecycle_methods(node) end methods = T.let([], T::Array[MIR::FnDef]) - if drop_statements.any? - drop_statements.unshift(MIR::Suppress.new("alloc")) - methods << MIR::FnDef.new( - "__clear_drop", - [MIR::Param.new("self", "*@This()", false), MIR::Param.new("alloc", "std.mem.Allocator", false)], - "void", - drop_statements, - :pub, - false, - [], - ) - end - if clone_fields.any? && !copy_forbidden + # Emitted even when no field owns anything. `__clear_drop` is the type's + # ownership contract, and cleanup consults it BEFORE falling back to + # representation-driven reflection -- which cannot tell an owned String from + # a `String@symbol`, a `@rodata` literal, or a borrow, since all four are + # []const u8, and frees the static behind the last three. A struct that owns + # nothing has to say so rather than say nothing. + drop_statements.unshift(MIR::Suppress.new("alloc")) + drop_statements.unshift(MIR::Suppress.new("self")) if drop_statements.length == 1 + methods << MIR::FnDef.new( + "__clear_drop", + [MIR::Param.new("self", "*@This()", false), MIR::Param.new("alloc", "std.mem.Allocator", false)], + "void", + drop_statements, + :pub, + false, + [], + ) + # Drop and clone are ONE contract: the runtime reads drop-without-clone as + # "linear" and rejects the copy, so a copyable struct carries both. + unless copy_forbidden + # A struct with nothing to clone never writes to `result`, and Zig rejects + # a `var` that is never mutated. + clone_statements[0] = MIR::Let.new("result", self_ref, clone_fields.any?, nil, nil, nil) + clone_statements.unshift(MIR::Suppress.new("alloc")) clone_statements << MIR::ReturnStmt.new(MIR::Ident.new("result")) methods << MIR::FnDef.new( "__clear_clone", @@ -3946,7 +4169,7 @@ def lower_inline_union_helper_struct(fact) deinit_entries.each do |de| tmp_name = "__dupe_#{de.field}" - field_source = "self.#{de.field}" + field_source = "self.#{zig_safe_name(de.field.to_s)}" dupe_stmts << MIR::Let.new( tmp_name, MIR::Lit.new("try CheatLib.dupeValue(@TypeOf(#{field_source}), #{field_source}, alloc)"), @@ -4041,13 +4264,19 @@ def lower_union_lifecycle_methods(node, facts) cleanup_arms = T.let([], T::Array[MIR::UnionMatchArm]) needs_cleanup = T.let(false, T::Boolean) copy_forbidden = T.let(false, T::Boolean) - clone_arms = T.let([], T::Array[String]) + clone_arms = T.let([], T::Array[MIR::UnionMatchArm]) facts.each do |fact| data = fact.data if data.nil? cleanup_arms << MIR::UnionMatchArm.new(variant: fact.name, payload: nil, body: []) - clone_arms << ".#{fact.name} => .{ .#{fact.name} = {} }" + clone_arms << MIR::UnionMatchArm.new( + variant: fact.name, + payload: nil, + body: [MIR::ReturnStmt.new( + MIR::StructInit.new(nil, [{ name: fact.name, value: MIR::Lit.new("void{}") }]), + )], + ) next end @@ -4056,16 +4285,22 @@ def lower_union_lifecycle_methods(node, facts) copy_strategy = T.let(:bit_copy, Symbol) if fact.inline_struct inline = T.cast(data, Schemas::InlineStructVariant) - if inline.deinit_entries.any? - body << MIR::ExprStmt.new( - emit_builtin(:cleanup, [MIR::Ident.new(fact.zig_type), MIR::Ident.new("alloc"), MIR::Ident.new(payload)]), - false, - ) - end + # `deinit_entries` covers a payload that closes a resource. It does not + # see a field that owns only through a capability -- an `@shared` field + # is an Arc whose refcount this drop has to release -- so the registry + # is what decides, exactly as it does for a non-inline variant. + drop_payload = T.let(inline.deinit_entries.any?, T::Boolean) inline.fields.each_value do |field| plan = lifecycle_registry.fetch(Type.from_input(field)) copy_forbidden ||= plan.copy_strategy == :forbidden copy_strategy = :deep_clone if plan.copy_strategy != :bit_copy + drop_payload ||= plan.needs_drop? + end + if drop_payload + body << MIR::ExprStmt.new( + emit_builtin(:cleanup, [MIR::Ident.new(fact.zig_type), MIR::Ident.new("alloc"), MIR::Ident.new(payload)]), + false, + ) end else variant_type = Type.from_variant_input(data) @@ -4083,12 +4318,21 @@ def lower_union_lifecycle_methods(node, facts) end needs_cleanup ||= body.any? + # The arm captures by pointer: a by-value capture would put a full copy of + # every variant's payload on the frame, once per arm. clone_payload = if copy_strategy == :bit_copy - payload + "#{payload}.*" else - "try CheatLib.dupeValue(@TypeOf(#{payload}), #{payload}, alloc)" + "try CheatLib.dupeValue(@TypeOf(#{payload}.*), #{payload}.*, alloc)" end - clone_arms << ".#{fact.name} => |#{payload}| .{ .#{fact.name} = #{clone_payload} }" + clone_arms << MIR::UnionMatchArm.new( + variant: fact.name, + payload: payload, + pointer_payload: true, + body: [MIR::ReturnStmt.new( + MIR::StructInit.new(nil, [{ name: fact.name, value: MIR::Lit.new(clone_payload) }]), + )], + ) cleanup_arms << MIR::UnionMatchArm.new( variant: fact.name, payload: payload, @@ -4098,28 +4342,38 @@ def lower_union_lifecycle_methods(node, facts) end methods = T.let([], T::Array[MIR::FnDef]) - if needs_cleanup - statements = T.let([ - MIR::Suppress.new("alloc"), - MIR::UnionMatchStmt.new(MIR::Deref.new(MIR::Ident.new("self")), cleanup_arms, nil), - ], T::Array[MIR::Stmt]) - methods << MIR::FnDef.new( - "__clear_drop", - [MIR::Param.new("self", "*@This()", false), MIR::Param.new("alloc", "std.mem.Allocator", false)], - "void", - statements, - :pub, - false, - [], - ) - end - if needs_cleanup && !copy_forbidden - clone_expr = "switch (self) { #{clone_arms.join(', ')}, }" + # Emitted even when no variant owns anything. `__clear_drop` is the type's + # ownership contract, and cleanup consults it BEFORE falling back to + # representation-driven reflection -- which cannot tell an owned String + # from a `String@symbol` or a borrow, since all three are []const u8, and + # frees the rodata behind a symbol. A union that owns nothing has to say so. + statements = T.let([ + MIR::Suppress.new("alloc"), + MIR::UnionMatchStmt.new(MIR::Deref.new(MIR::Ident.new("self")), cleanup_arms, nil), + ], T::Array[MIR::Stmt]) + methods << MIR::FnDef.new( + "__clear_drop", + [MIR::Param.new("self", "*@This()", false), MIR::Param.new("alloc", "std.mem.Allocator", false)], + "void", + statements, + :pub, + false, + [], + ) + # Drop and clone are ONE contract: the runtime reads drop-without-clone as + # "linear" and rejects the copy. Now that drop is unconditional, a copyable + # union has to carry its clone too, whether or not a variant owns anything. + unless copy_forbidden + # A switch EXPRESSION gives every arm its own result temp. On a union with + # a hundred variants that is megabytes of frame -- enough to blow a 4 MB + # fiber stack in the prologue. Returning from each arm reuses the return + # slot instead. methods << MIR::FnDef.new( "__clear_clone", [MIR::Param.new("self", "@This()", false), MIR::Param.new("alloc", "std.mem.Allocator", false)], "@This()", - [MIR::ReturnStmt.new(MIR::Lit.new(clone_expr))], + # A union whose variants all bit-copy never touches the allocator. + [MIR::Suppress.new("alloc"), MIR::UnionMatchStmt.new(MIR::Ident.new("self"), clone_arms, nil)], :pub, true, [], @@ -4157,11 +4411,30 @@ def union_variant_lowering_facts(node) # - cheat_runtime: CLEAR runtime, wired via build.zig as a module EXTERN_MODULE_ROOTS = T.let(%w[std builtin cheat_runtime].to_set.freeze, T::Set[String]) - sig { params(node: AST::Cast).returns(MIR::Cast) } + # Returns MIR::Node, not MIR::Cast: widening `CAST(sym AS String)` is a + # field read of the interned handle, and a noreturn value passes through + # unchanged. + sig { params(node: AST::Cast).returns(MIR::Node) } def lower_cast(node) inner = lower(node.value) + # A NoReturn value coerces to every type in Zig; `@as(T, @panic(...))` is + # unreachable code at the use site. `CAST(panic("...") AS T)` is how the + # translation spells an unreachable fallback. + return inner if Hoist.noreturn_value?(node.value) + target_type = transpile_type(node.target) + # `CAST(sym AS String)` IS the widening from an interned handle to the + # bytes behind it -- not a coercion Zig can do, now that Symbol is its own + # type. Read the field instead of casting. The condition is the TYPE's, + # not the rendered Zig string's: semantic decisions in lowering come from + # Type stamps (INV-7 territory), and two types may render alike. + cast_target = Type.new(node.target) + if cast_target.byte_string? && !cast_target.optional? + widened = widen_symbol_to_bytes(inner, node.value) + return widened unless widened.equal?(inner) + end + # Int -> enum: emit `@enumFromInt(value)` instead of `@as(EnumT, value)`. # Modern Zig rejects `@as(EnumT, intExpr)` (type coercion is enum-from- # int, which is its own builtin). Detected by checking whether the @@ -4253,7 +4526,11 @@ def lower_require(node) pkg_inline = node.kind == :package && importer && importer.stdlib_package?(node.path) if node.kind == :package && !pkg_inline + # Only the OWNING package of a multi-file group is built, so both the + # import and the type aliases below must name it -- otherwise the same + # CLEAR type reaches Zig as two distinct types. import_name = node.namespace || node.path + import_name = importer.owning_package_name(import_name) if importer && importer.respond_to?(:owning_package_name) zig_import_name = zig_module_alias(import_name) # The same package can be required by the root and by an inlined local # module; both land in one Zig compilation unit, so emit each import @@ -4264,6 +4541,12 @@ def lower_require(node) # each pub type to the imported module in the emitted Zig — the same # contract EXTERN STRUCT already emits for foreign types. pkg_mod = importer&.compile_package(node.path, caller_dir: T.must(program_state.source_dir)) + # MATCH dispatch reads union_schemas to decide switch-with-payload vs a + # tag equality chain. Without the imported package's schemas an + # `Imported.Variant AS payload` arm silently lowered to `value == + # Imported.Variant` and never bound the payload. + merge_module_schemas!(pkg_mod) if pkg_mod + record_module_const_init!(pkg_mod, zig_import_name) pkg_scope = pkg_mod&.global_scope if pkg_scope alias_target = zig_import_name @@ -4356,6 +4639,17 @@ def imported_module_extern_items(mod) end.flatten.select { |item| item.is_a?(MIR::Emittable) } end + # A module's runtime-initialized consts sit in its own Zig file as + # `undefined` until its initializer runs. Remember the ones that have an + # initializer so the root can call it before anything reads them. + sig { params(mod: T.nilable(ModuleImporter::CompiledModule), alias_name: String).void } + def record_module_const_init!(mod, alias_name) + items = mod&.mir_items + return unless items + has_init = items.flatten.any? { |item| item.is_a?(MIR::FnDef) && item.name.to_s == CONST_INIT_FN } + program_state.module_const_inits << alias_name if has_init + end + sig { params(mod: ModuleImporter::CompiledModule).void } def merge_module_schemas!(mod) struct_schemas = mod.struct_schemas @@ -4561,7 +4855,7 @@ def lower_struct_pattern(subject, pat) sig { params(node: AST::FuncCall).returns(MIR::Call) } def lower_macro_print(node) formats = node.args.map { |arg| zig_format_for_type(arg.full_type!) }.join(" ") - args_mir = node.args.map { |a| hoist_alloc(lower(a), a) } + args_mir = node.args.map { |a| hoist_alloc(widen_symbol_to_bytes(lower(a), a), a) } format_lit = MIR::Lit.new("\"#{formats}\\n\"") tuple = MIR::TupleLiteral.new(args_mir) MIR::Call.new("std.debug.print", [format_lit, tuple], false, false, MIR::CallableContract.no_ownership(2)) @@ -4651,7 +4945,8 @@ def lower_direct_length(node) recv = hoist_alloc(recv, recv_ast) if mir_allocates?(recv) return nil unless ti.string? - MIR::Cast.new(MIR::ListLength.new(recv), "i64", :intCast) + # `.len` reads bytes; a Symbol receiver widens like any String position. + MIR::Cast.new(MIR::ListLength.new(widen_symbol_to_bytes(recv, recv_ast)), "i64", :intCast) end # Rc/Arc capability values expose ordinary methods and TAKES boundaries in @@ -4975,6 +5270,20 @@ def owned_sink_plan(value, ast_node, sink_alloc, sink_type = nil) raise "annotation admitted an implicit copy of linear type #{lifecycle.type_key}" end + # A destination that IS an Rc/Arc handle is filled by retaining, whatever + # the lifecycle plan of the surface type says: a structural copy of a + # handle fabricates an owner that was never counted. + handle_ti = dst_ti.optional? ? dst_ti.wrapped_type : dst_ti + if handle_ti&.any_rc? && !source.satisfies_rc_sink? + return OwnedSinkPlan.new( + action: :rc_retain, + target_alloc: sink_alloc, + zig_type: rc_payload_zig_type(handle_ti), + copy_mode: nil, + rc_func: handle_ti.shared? ? "arcRetain" : "rcRetain", + ) + end + if lifecycle.copy_strategy == :deep_clone || lifecycle.copy_strategy == :generic if source.borrowed_union_sink return OwnedSinkPlan.new( diff --git a/compiler/ruby/mir/rewriters/pipeline_rewriter.rb b/compiler/ruby/mir/rewriters/pipeline_rewriter.rb index fd4a293ee..1d85d363a 100644 --- a/compiler/ruby/mir/rewriters/pipeline_rewriter.rb +++ b/compiler/ruby/mir/rewriters/pipeline_rewriter.rb @@ -595,11 +595,14 @@ def build_init(terminal, res_var, token, smooth_node) # A heap destination (see rewrite_children! VarDecl) owns the accumulator # wholesale; otherwise keep the pipeline expression's own stamp. decl.storage = @dest_storage == :heap ? :heap : smooth_node.storage - if @dest_storage == :heap - sym = SymbolEntry.new(reg: decl, type: Type.new(decl.full_type!), mutable: true, storage: :heap) - decl.symbol = sym - @list_res_symbols[res_var] = sym - end + # The accumulator gets a SymbolEntry whatever its initial placement, and + # every reference shares it. Escape analysis runs AFTER this rewrite and + # promotes bindings through value-block results; without a symbol to + # promote, an accumulator feeding a heap binding stayed frame-allocated + # (OWNED_RESULT_ALLOC_MISMATCH). + sym = SymbolEntry.new(reg: decl, type: Type.new(decl.full_type!), mutable: true, storage: T.must(decl.storage)) + decl.symbol = sym + @list_res_symbols[res_var] = sym decl.slot_size = Type.new(decl.full_type!).slot_size(T.unsafe(schema_lookup)) decl.var_used = true [decl] @@ -877,7 +880,7 @@ def build_final_result(terminal, res_var, token, smooth_node) AST.stamp_synthetic_type!(res, smooth_node.full_type!, context: "synthetic AST type") if (sym = @list_res_symbols[res_var]) res.symbol = sym - res.storage = :heap + res.storage = sym.storage else res.storage = smooth_node.storage end diff --git a/compiler/ruby/mir/rewriters/string_concat_rewriter.rb b/compiler/ruby/mir/rewriters/string_concat_rewriter.rb index c5d2f01a3..88b934631 100644 --- a/compiler/ruby/mir/rewriters/string_concat_rewriter.rb +++ b/compiler/ruby/mir/rewriters/string_concat_rewriter.rb @@ -66,7 +66,7 @@ def rewrite_required_node!(node) def rewrite_body!(body) index = 0 while index < body.length - body[index] = rewrite_required_node!(T.must(body[index])) + body[index] = rewrite_required_node!(body.fetch(index)) index += 1 end end diff --git a/compiler/ruby/semantic/escape_analysis.rb b/compiler/ruby/semantic/escape_analysis.rb index 68ecdc997..b1baa4fae 100644 --- a/compiler/ruby/semantic/escape_analysis.rb +++ b/compiler/ruby/semantic/escape_analysis.rb @@ -135,14 +135,15 @@ class EscapeSink < T::Struct sig { params(node: BasicObject).returns(T::Boolean) } def matches?(node) - case handler - when :apply_return_escape_sink! then T.unsafe(node).is_a?(AST::ReturnNode) - when :apply_assignment_escape_sink! then T.unsafe(node).is_a?(AST::Assignment) - when :apply_binding_escape_sink! then T.unsafe(node).is_a?(AST::VarDecl) || T.unsafe(node).is_a?(AST::BindExpr) - when :apply_execution_boundary_escape_sink! then T.unsafe(node).is_a?(AST::BgBlock) || T.unsafe(node).is_a?(AST::BgStreamBlock) - when :apply_lambda_escape_sink! then T.unsafe(node).is_a?(AST::LambdaLit) - when :apply_func_call_escape_sink! then T.unsafe(node).is_a?(AST::FuncCall) - when :apply_method_call_escape_sink! then T.unsafe(node).is_a?(AST::MethodCall) + case node + when AST::ReturnNode then handler == :apply_return_escape_sink! + when AST::Assignment then handler == :apply_assignment_escape_sink! + when AST::VarDecl, AST::BindExpr then handler == :apply_binding_escape_sink! + when AST::DestructuringAssignment then handler == :apply_destructuring_escape_sink! + when AST::BgBlock, AST::BgStreamBlock then handler == :apply_execution_boundary_escape_sink! + when AST::LambdaLit then handler == :apply_lambda_escape_sink! + when AST::FuncCall then handler == :apply_func_call_escape_sink! + when AST::MethodCall then handler == :apply_method_call_escape_sink! else false end end @@ -174,6 +175,7 @@ def matches?(node) :owning_return, :enclosing_scope_store, :binding_result, + :destructured_binding, :execution_boundary_capture, :lambda_capture, :takes_or_mutable_arg, @@ -191,6 +193,7 @@ def matches?(node) :apply_return_escape_sink!, :apply_assignment_escape_sink!, :apply_binding_escape_sink!, + :apply_destructuring_escape_sink!, :apply_execution_boundary_escape_sink!, :apply_lambda_escape_sink!, :apply_func_call_escape_sink!, @@ -210,6 +213,7 @@ def matches?(node) EscapeSink.new(name: :owning_return, node_classes: [AST::ReturnNode], handler: :apply_return_escape_sink!), EscapeSink.new(name: :enclosing_scope_store, node_classes: [AST::Assignment], handler: :apply_assignment_escape_sink!), EscapeSink.new(name: :binding_result, node_classes: [AST::VarDecl, AST::BindExpr], handler: :apply_binding_escape_sink!), + EscapeSink.new(name: :destructured_binding, node_classes: [AST::DestructuringAssignment], handler: :apply_destructuring_escape_sink!), EscapeSink.new(name: :execution_boundary_capture, node_classes: [AST::BgBlock, AST::BgStreamBlock], handler: :apply_execution_boundary_escape_sink!), EscapeSink.new(name: :lambda_capture, node_classes: [AST::LambdaLit], handler: :apply_lambda_escape_sink!), EscapeSink.new(name: :takes_or_mutable_arg, node_classes: [AST::FuncCall], handler: :apply_func_call_escape_sink!), @@ -415,13 +419,18 @@ def self.propagate_caller_sync!(fn_nodes, body_summaries) body_summaries: BodySummaries, on_mutable_violation: T.nilable(T.proc.params(entry: SymbolEntry, arg: AST::Identifier, callee_name: String).void), on_family_violation: T.nilable(T.proc.params(arg: AST::Node, source_family: Symbol, dest_family: Symbol, callee_name: String).void), + imported_params: T::Hash[String, T::Array[AST::Param]], ).void end - def self.apply_kept_identity_placement!(fn_nodes, body_summaries, on_mutable_violation: nil, on_family_violation: nil) + def self.apply_kept_identity_placement!(fn_nodes, body_summaries, on_mutable_violation: nil, on_family_violation: nil, imported_params: {}) kept_contracts = T.let({}, T::Hash[String, T::Hash[Integer, KeptIdentityContract]]) - fn_nodes.each do |name, fn| + # An imported callee is not in fn_nodes, but it keeps its arguments just + # the same. Without its contract the call site gets no edge plan, so the + # caller hands over an Rc without a retain. + all_params = imported_params.merge(fn_nodes.transform_values(&:params)) + all_params.each do |name, params| by_index = T.let({}, T::Hash[Integer, KeptIdentityContract]) - fn.params.each_with_index do |param, idx| + params.each_with_index do |param, idx| entry = param.symbol contract = entry&.kept_identity next unless entry && contract @@ -709,6 +718,7 @@ def index_node(node, loop_depth) when :apply_return_escape_sink! then apply_return_escape_sink!(T.cast(node, AST::ReturnNode), context) when :apply_assignment_escape_sink! then apply_assignment_escape_sink!(T.cast(node, AST::Assignment), context) when :apply_binding_escape_sink! then apply_binding_escape_sink!(T.cast(node, T.any(AST::VarDecl, AST::BindExpr)), context) + when :apply_destructuring_escape_sink! then apply_destructuring_escape_sink!(T.cast(node, AST::DestructuringAssignment), context) when :apply_execution_boundary_escape_sink! then apply_execution_boundary_escape_sink!(T.cast(node, T.any(AST::BgBlock, AST::BgStreamBlock)), context) when :apply_lambda_escape_sink! then apply_lambda_escape_sink!(T.cast(node, AST::LambdaLit), context) when :apply_func_call_escape_sink! then apply_func_call_escape_sink!(T.cast(node, AST::FuncCall), context) @@ -746,6 +756,22 @@ def index_node(node, loop_depth) end end + # `_, items = call()` hands each target a piece of an aggregate the callee + # allocated on the heap. A destructuring target records no VALUE, only its + # symbol, so nothing else places it -- a target declared earlier keeps the + # frame allocation its empty-literal initialiser chose. + sig { params(node: AST::DestructuringAssignment, context: EscapeContext).void } + private_class_method def self.apply_destructuring_escape_sink!(node, context) + return unless node.value + + node.targets.each do |target| + next if target.name.to_s == "_" + ti = Type.from_node!(target, context: "destructured target placement") + next unless ti.needs_explicit_cleanup?(:heap, T.unsafe(context.schema_lookup)) + mark_symbol_heap!(target.symbol) + end + end + sig { params(facts: FunctionFacts).void } private_class_method def self.propagate_value_block_placements!(facts) body = facts.fn.body @@ -854,7 +880,8 @@ def index_node(node, loop_depth) private_class_method def self.aggregate_contains_heap_owned_value?(node) return false unless node return false unless node.is_a?(AST::StructLit) || node.is_a?(AST::UnionVariantLit) || - node.is_a?(AST::ListLit) || node.is_a?(AST::HashLit) + node.is_a?(AST::ListLit) || node.is_a?(AST::HashLit) || + node.is_a?(AST::TupleLit) pending = T.let([node], T::Array[AST::Node]) until pending.empty? @@ -1208,7 +1235,8 @@ def index_node(node, loop_depth) node = unwrap_value(node) return true if node.is_a?(AST::Identifier) return true if node.is_a?(AST::StructLit) || node.is_a?(AST::UnionVariantLit) || - node.is_a?(AST::ListLit) || node.is_a?(AST::HashLit) + node.is_a?(AST::ListLit) || node.is_a?(AST::HashLit) || + node.is_a?(AST::TupleLit) false end @@ -1520,6 +1548,11 @@ def index_node(node, loop_depth) sig { params(facts: FunctionFacts, expr: AST::Node).void } private_class_method def self.mark_heap_return!(facts, expr) fn = facts.fn + # `RETURNS self: T` declares the result a borrow scoped to a parameter, so + # the caller's argument already owns the storage. Promoting anything here + # would fabricate an owned result out of a view. + return unless Array(fn.return_lifetime).empty? + ret = fn.declared_return_type ret = ret.value_payload_type if ret ret.mark_heap_allocated! if ret diff --git a/compiler/ruby/semantic/lifecycle_plan.rb b/compiler/ruby/semantic/lifecycle_plan.rb index 319790111..1a76dd0e4 100644 --- a/compiler/ruby/semantic/lifecycle_plan.rb +++ b/compiler/ruby/semantic/lifecycle_plan.rb @@ -202,12 +202,22 @@ def self.plan(type_info, schema_lookup, linear_resource_facts = nil) return LifecyclePlan.new(type_key: type_key, drop_strategy: :none, copy_strategy: :bit_copy) end + # An Rc/Arc CARRIER is itself the owned thing: a handle is released by + # decrementing its refcount, and where its payload came from does not + # change that. Only the payload can be a borrow. + if type_info.any_rc? && !type_info.rodata? + return LifecyclePlan.new(type_key: type_key, drop_strategy: :release, copy_strategy: :retain) + end + if type_info.borrowed_reference? || type_info.rodata? copy = if resource_facts.contains?(type_info) :forbidden elsif type_info.any_rc? :retain - elsif type_info.string? || type_info.recursive_cleanup_shape?(schema_lookup) + elsif type_info.string? || type_info.recursive_cleanup_shape?(schema_lookup, nil, ignore_borrow: true) + # COPY through a borrow duplicates what the POINTEE owns. A bit copy + # here aliases the pointee's heap fields into a second value that is + # then cleaned up independently. :deep_clone else :bit_copy diff --git a/compiler/ruby/semantic/tense_operation_plan.rb b/compiler/ruby/semantic/tense_operation_plan.rb index e2463d76e..95b2ebe47 100644 --- a/compiler/ruby/semantic/tense_operation_plan.rb +++ b/compiler/ruby/semantic/tense_operation_plan.rb @@ -621,6 +621,16 @@ def self.or_else(type, fallback_type, operation: TenseOperationKind::OrElseValue remaining = envelope.layers.drop(handled.length) result = Type.new(TenseEnvelope.wrap_layers(envelope.payload_expression, remaining)) + # OR_ELSE strips tense layers, not capabilities: the payload expression + # does not carry the source's sync/collection/ownership, so rebuilding from + # it alone turns `?String@symbol` into a plain String. + result.merge_capabilities_from!(type, include_affine_ownership: true) + # A fallback that is itself optional cannot make the result definite: + # `a OR_ELSE b` with both absent is still absent. Typing it as the payload + # makes downstream placement copy a null as though it were present. + if recovery == TenseRecovery::Fallback && fallback_type.optional? && !result.optional? + result = Type.optional_of(result) + end if recovery == TenseRecovery::Fallback && fallback_type.resolved != :NoReturn && !result.accepts?(fallback_type) && !fallback_type.accepts?(result) raise ArgumentError, "OR_ELSE fallback #{fallback_type.resolved} does not match #{result.resolved}" diff --git a/compiler/ruby/tools/clear_build_support.rb b/compiler/ruby/tools/clear_build_support.rb index ac09cf2b5..1d3079f25 100644 --- a/compiler/ruby/tools/clear_build_support.rb +++ b/compiler/ruby/tools/clear_build_support.rb @@ -55,6 +55,56 @@ def self.write_if_changed(path, content) true end + # The per-program build caches live on a tmpfs that fills after a few dozen + # large builds, and a full cache is reported as `DWARF TODO: 'NoSpaceLeft'` + # rather than as a disk error. Drop the ones nobody has touched in a while. + # + # Age, not entry count, is the only safe criterion: parallel workers each + # build their own cache key, so a count-based rule deletes a directory + # another process is building into and that process then fails to symlink + # into its own cache. + CACHE_ENTRY_MAX_AGE_SECONDS = 3600 + + sig { params(cache_root: String, keep: String).void } + def self.prune_build_cache!(cache_root, keep:) + keep_real = File.expand_path(keep) + # This build is using its directory now, whether or not it just created it. + FileUtils.touch(keep) if File.directory?(keep) + cutoff = Time.now - CACHE_ENTRY_MAX_AGE_SECONDS + Dir.glob(File.join(cache_root, '*')).each do |path| + next unless File.directory?(path) + next if File.expand_path(path) == keep_real + next if File.mtime(path) > cutoff + + FileUtils.rm_rf(path) + end + rescue StandardError + # Pruning is opportunistic; a build must never fail because of it. + nil + end + + # A build killed mid-run (ENOSPC, timeout, SIGKILL) never reaches its + # rm_rf, so `.build-` directories accumulate. Reap the ones whose + # process is gone. + sig { params(zig_dir: String).void } + def self.reap_orphan_build_dirs!(zig_dir) + Dir.glob(File.join(zig_dir, '.build-*')).each do |path| + pid = File.basename(path).delete_prefix('.build-').to_i + next if pid <= 0 || pid == Process.pid + + begin + Process.kill(0, pid) + next # still running + rescue Errno::ESRCH + FileUtils.rm_rf(path) + rescue Errno::EPERM + next # someone else's process + end + end + rescue StandardError + nil + end + sig { params(link_path: String, target_path: String).void } def self.ensure_symlink(link_path, target_path) if File.symlink?(link_path) @@ -174,6 +224,46 @@ def self.materialize_package_root(pkg_name) out end + # Run `block` over `items` in forked workers purely for its cache side + # effects, then return. Every expensive step behind it -- transpile_cached, + # the module cache -- is content-addressed on disk, so a child populates + # exactly what the serial pass that follows will look up. Nothing is read + # back from the children, which is what keeps this to one call site instead + # of threading results through the build. + # + # Falls back to serial when jobs <= 1 or fork is unavailable. + sig do + params(items: T::Array[T.untyped], jobs: Integer, block: T.proc.params(item: T.untyped).void).void + end + def self.prewarm_in_parallel(items, jobs:, &block) + return if items.empty? + if jobs <= 1 || !Process.respond_to?(:fork) + items.each { |item| block.call(item) } + return + end + + queue = items.dup + running = T.let({}, T::Hash[Integer, T::Boolean]) + until queue.empty? && running.empty? + while running.size < jobs && !queue.empty? + item = queue.shift + pid = Process.fork do + begin + block.call(item) + rescue StandardError, SystemExit + # A warm-up failure is never fatal: the serial pass re-runs the + # same work and reports the real diagnostic in the right order. + end + Process.exit!(0) + end + running[pid] = true + end + pid, _ = Process.wait2 + running.delete(pid) + end + nil + end + sig { params(pkg_name: String, start_dir: String).returns(T.nilable(String)) } def self.find_package_source(pkg_name, start_dir:) registered = @registered_packages[pkg_name] diff --git a/compiler/ruby/tools/predicate_rewriter.rb b/compiler/ruby/tools/predicate_rewriter.rb index 4f528c9b8..3f5e75c59 100644 --- a/compiler/ruby/tools/predicate_rewriter.rb +++ b/compiler/ruby/tools/predicate_rewriter.rb @@ -350,6 +350,13 @@ def self.leftmost_offset(node, source) case node when AST::MethodCall leftmost_offset(node.object, source) + when AST::GetIndex + # A GetIndex/GetField carries the `[` / `.` token, not the receiver's, + # so the span would start mid-expression and the rewrite would orphan + # the receiver (`m[:a]` became `m([:a])`). + leftmost_offset(node.target, source) + when AST::GetField + leftmost_offset(node.target, source) when AST::FuncCall offset_for(source, node.token.line, node.token.column) if node.token else diff --git a/compiler/spec/annotator_spec.rb b/compiler/spec/annotator_spec.rb index 672d0c2f2..2fd1733e7 100644 --- a/compiler/spec/annotator_spec.rb +++ b/compiler/spec/annotator_spec.rb @@ -4090,7 +4090,7 @@ def transpile_map(clear_src) RETURN; END CLEAR - expect(out).to include("__hm.put(__clear_heap_alloc, __clear_heap_alloc") + expect(out).to match(/__hm_\d+\.put\(__clear_heap_alloc, __clear_heap_alloc/) expect(out).to include('"a"') expect(out).to include('"b"') end diff --git a/compiler/spec/binary_operator_type_check_spec.rb b/compiler/spec/binary_operator_type_check_spec.rb index 9cb877e2a..149527497 100644 --- a/compiler/spec/binary_operator_type_check_spec.rb +++ b/compiler/spec/binary_operator_type_check_spec.rb @@ -44,7 +44,10 @@ def expect_reject_expr(expr, returns: "Bool") it "reserves + for numeric addition and $+ for string concatenation" do expect_reject_expr('"a" + "b"', returns: "String") expect_reject_expr('1 $+ 2', returns: "String") - expect(Type.binary_op(:CONCAT, Type.new(:String), Type.new(:Int64)).type.resolved).to eq(:String) + # A number has no bit-level coercion to a string; the emitter could only + # render it as `@as([]const u8, n)`, which is not valid Zig. + expect(Type.binary_op(:CONCAT, Type.new(:String), Type.new(:Int64)).error) + .to include("call .toString()") end it "accepts valid boolean logic" do diff --git a/compiler/spec/caller_cleanup_spec.rb b/compiler/spec/caller_cleanup_spec.rb index 959ffb17d..e9d5fd5ee 100644 --- a/compiler/spec/caller_cleanup_spec.rb +++ b/compiler/spec/caller_cleanup_spec.rb @@ -10,8 +10,18 @@ def transpile(src) ZigTranspiler.new.transpile(src) end + # Emitted bodies now contain nested `{ ... }` scopes whose closing brace sits + # at column 0, so stopping at the FIRST line-start `}` truncates the body. + # Take everything up to the last one before the next top-level fn instead. def fn_body(zig, name) - zig[/fn #{name}\b.*?\n(.*?)^}/m, 1] + start = zig.index(/^(?:pub )?fn #{Regexp.escape(name)}\b/) + return nil unless start + + rest = zig[start..] + after = rest[(rest.index("\n") + 1)..] + nxt = after.index(/^(?:pub )?fn \w/) + segment = nxt ? after[0...nxt] : after + segment[/\A(.*)^}/m, 1] || segment end # ========================================================================= diff --git a/compiler/spec/clear_build_support_spec.rb b/compiler/spec/clear_build_support_spec.rb index 8a8a8fa94..40d0109e7 100644 --- a/compiler/spec/clear_build_support_spec.rb +++ b/compiler/spec/clear_build_support_spec.rb @@ -512,4 +512,28 @@ def simple_signature(config, source, output, extra_flags: ["-fno-llvm"]) expect(described_class.run_transpiler(failing_config, "", source)).to be_nil end end + # Parallel workers each build their own cache key into the same root, so a + # rule that drops all but the newest N entries deletes a directory another + # process is building into -- that process then fails to symlink the runtime + # into its own cache with ENOENT. + it "prunes only build-cache entries nobody has touched for an hour" do + Dir.mktmpdir do |root| + crowd = 12.times.map { |i| File.join(root, format("key%02d", i)) } + crowd.each { |path| FileUtils.mkdir_p(path) } + described_class.prune_build_cache!(root, keep: crowd.first) + expect(crowd.select { |path| File.directory?(path) }).to eq(crowd) + + stale = File.join(root, "stale") + FileUtils.mkdir_p(stale) + touch_at(stale, Time.now.to_i - ClearBuildSupport::CACHE_ENTRY_MAX_AGE_SECONDS - 60) + keep = File.join(root, "keep") + FileUtils.mkdir_p(keep) + touch_at(keep, Time.now.to_i - ClearBuildSupport::CACHE_ENTRY_MAX_AGE_SECONDS - 60) + + described_class.prune_build_cache!(root, keep: keep) + + expect(File.directory?(stale)).to be(false) + expect(File.directory?(keep)).to be(true) + end + end end diff --git a/compiler/spec/comptime_if_spec.rb b/compiler/spec/comptime_if_spec.rb index 04775ca43..3d20038a0 100644 --- a/compiler/spec/comptime_if_spec.rb +++ b/compiler/spec/comptime_if_spec.rb @@ -63,7 +63,9 @@ def transpile(source) CLEAR expect(zig).to include("fn handle(comptime T: type, x: T)") - expect(zig).to include("if (comptime (T == []const u8))") + # `String@symbol` is its own Zig type, so the predicate can finally tell a + # symbol from a String -- against []const u8 it matched both. + expect(zig).to include("if (comptime (T == CheatLib.Symbol))") end it "allows a then-branch type binding" do diff --git a/compiler/spec/error_registry_spec.rb b/compiler/spec/error_registry_spec.rb index 707bc65b1..41452bef6 100644 --- a/compiler/spec/error_registry_spec.rb +++ b/compiler/spec/error_registry_spec.rb @@ -274,4 +274,33 @@ end end end + + # A module compiles to its own Zig file, so a RAISE inside it names + # `ErrorName.` with no definition in scope unless the module emits the + # enum too. Only the root program used to. + describe "module emission" do + require_relative "../ruby/backends/transpiler" unless defined?(ZigTranspiler) + + it "emits the ErrorName enum in a module that raises" do + out = ZigTranspiler.new.transpile_as_module(<<~CLEAR) + PUB FN check(limit: Int64) RETURNS !Void -> + IF (limit > 10_i64) THEN + RAISE Input, Exceeded, "over the limit"; + END + RETURN; + END + CLEAR + expect(out).to include("pub const ErrorName = enum(u32) {") + expect(out).to include("ErrorName.Exceeded") + end + + it "omits the enum from a module that never names one" do + out = ZigTranspiler.new.transpile_as_module(<<~CLEAR) + PUB FN double(value: Int64) RETURNS Int64 -> + RETURN (value * 2_i64); + END + CLEAR + expect(out).not_to include("pub const ErrorName") + end + end end diff --git a/compiler/spec/generic_associated_map_storage_spec.rb b/compiler/spec/generic_associated_map_storage_spec.rb index 6a0f30391..5daf772f9 100644 --- a/compiler/spec/generic_associated_map_storage_spec.rb +++ b/compiler/spec/generic_associated_map_storage_spec.rb @@ -55,7 +55,10 @@ def transpile(source) END CLEAR - expect(zig).to include("ProjectionBox(Store(i64)){ .latest = @as(?i64, null) }") + expect(zig).to include("ProjectionBox(Store(i64)){ .latest = null }") + # The projection stays generic in the definition and resolves per + # specialization; the literal above names the specialization. + expect(zig).to include("latest: ?__clearProtocolFacts_Identity(S).Value,") end it "does not misreport mutable generic calls as unused synchronization" do diff --git a/compiler/spec/incremental/module_cache_spec.rb b/compiler/spec/incremental/module_cache_spec.rb new file mode 100644 index 000000000..624e25b55 --- /dev/null +++ b/compiler/spec/incremental/module_cache_spec.rb @@ -0,0 +1,132 @@ +# typed: false +# frozen_string_literal: true + +require "tmpdir" + +require_relative "../../ruby/incremental/module_cache" + +RSpec.describe Incremental::ModuleCache do + around do |example| + Dir.mktmpdir("module-cache-spec-") do |dir| + @dir = dir + example.run + end + end + + def write(name, contents) + path = File.join(@dir, name) + File.write(path, contents) + path + end + + def cache(compiler_key: "compiler-1") + described_class.new(dir: File.join(@dir, "cache"), compiler_key: compiler_key) + end + + it "recompiles a unit only when one of its own sources changes" do + source = write("leaf.clear", "one") + compiles = 0 + unit = -> { cache.fetch("leaf", [source]) { compiles += 1; "compiled:#{File.read(source)}" } } + + expect(unit.call).to eq("compiled:one") + expect(unit.call).to eq("compiled:one") + expect(compiles).to eq(1) + + File.write(source, "two") + expect(unit.call).to eq("compiled:two") + expect(compiles).to eq(2) + end + + it "invalidates a unit when a source it read transitively changes" do + leaf = write("leaf.clear", "leaf-one") + root = write("root.clear", "root") + compiles = { leaf: 0, root: 0 } + + build = lambda do + store = cache + store.fetch("root", [root]) do + compiles[:root] += 1 + inner = store.fetch("leaf", [leaf]) do + compiles[:leaf] += 1 + File.read(leaf) + end + "root(#{inner})" + end + end + + expect(build.call).to eq("root(leaf-one)") + expect(build.call).to eq("root(leaf-one)") + expect(compiles).to eq({ leaf: 1, root: 1 }) + + File.write(leaf, "leaf-two") + expect(build.call).to eq("root(leaf-two)") + expect(compiles).to eq({ leaf: 2, root: 2 }) + end + + it "still records a shared dependency the importer resolved from its own cache" do + leaf = write("leaf.clear", "leaf-one") + first = write("first.clear", "first") + second = write("second.clear", "second") + compiles = Hash.new(0) + + build = lambda do + store = cache + # Whatever compiles the leaf first wins; the importer serves the second + # request from memory without re-entering the cache. + seen = {} + compile_leaf = lambda do + return seen[:leaf] if seen.key?(:leaf) + + seen[:leaf] = store.fetch("leaf", [leaf]) { compiles[:leaf] += 1; File.read(leaf) } + end + one = store.fetch("first", [first]) { compiles[:first] += 1; compile_leaf.call } + two = store.fetch("second", [second]) do + compiles[:second] += 1 + store.reuse("leaf") + seen.key?(:leaf) ? seen[:leaf] : compile_leaf.call + end + [one, two] + end + + expect(build.call).to eq(%w[leaf-one leaf-one]) + expect(compiles).to eq({ leaf: 1, first: 1, second: 1 }) + + File.write(leaf, "leaf-two") + expect(build.call).to eq(%w[leaf-two leaf-two]) + expect(compiles).to eq({ leaf: 2, first: 2, second: 2 }) + end + + it "keeps generations apart so a compiler change never reuses old units" do + source = write("leaf.clear", "one") + compiles = 0 + build = ->(key) { cache(compiler_key: key).fetch("leaf", [source]) { compiles += 1; "compiled" } } + + build.call("compiler-1") + build.call("compiler-1") + build.call("compiler-2") + + expect(compiles).to eq(2) + end + + it "stores nothing when the unit fails to compile" do + source = write("leaf.clear", "one") + store = cache + + expect { store.fetch("leaf", [source]) { raise ArgumentError, "boom" } }.to raise_error(ArgumentError) + + compiled = store.fetch("leaf", [source]) { "recovered" } + expect(compiled).to eq("recovered") + end + + it "is off unless the environment names both a directory and a key" do + bare = ENV.to_h.reject { |name, _| [described_class::DIR_ENV, described_class::KEY_ENV].include?(name) } + stub_const("ENV", bare) + expect(described_class.from_env).to be_nil + + stub_const("ENV", bare.merge( + described_class::DIR_ENV => File.join(@dir, "cache"), + described_class::KEY_ENV => "compiler-1" + )) + expect(described_class.from_env).to be_a(described_class) + end +end diff --git a/compiler/spec/integration/fixtures/hostile_frontend/seed_20260715_case_1.clear.bin b/compiler/spec/integration/fixtures/hostile_frontend/seed_20260715_case_1.clear.bin new file mode 100644 index 000000000..597a6db29 --- /dev/null +++ b/compiler/spec/integration/fixtures/hostile_frontend/seed_20260715_case_1.clear.bin @@ -0,0 +1 @@ +i \ No newline at end of file diff --git a/compiler/spec/lifetimes_spec.rb b/compiler/spec/lifetimes_spec.rb index a72eb64f3..d9a4192f6 100644 --- a/compiler/spec/lifetimes_spec.rb +++ b/compiler/spec/lifetimes_spec.rb @@ -406,3 +406,75 @@ def get_last_type(source) end end end + +RSpec.describe "WITH alias capability" do + # declare_with_new_capability marks the SOURCE binding, but the body reads + # through the alias and Scope#is_restricted? answers per binding. Without the + # alias carrying the capability, borrowing through it -- calling a + # `RETURNS self: T` accessor -- was refused as MUTABLE_PARAM_NEEDS_RESTRICT. + it "carries the capability onto the WITH alias, not only its source" do + src = <<~CLEAR + STRUCT Box { items: []Int64, pos: Int64 } + + PUB FN box__at(self: Box) RETURNS self: Int64 + REQUIRES self: LOCAL + -> + WITH POLYMORPHIC self AS view { + RETURN UNWRAP (view.items[view.pos]); + } + END + + PUB FN box__step(MUTABLE self: Box) RETURNS Int64 + REQUIRES self: LOCAL + -> + WITH POLYMORPHIC self AS MUTABLE view { + IF box__at(view) == 0_i64 THEN + RETURN 0_i64; + END + RETURN 1_i64; + } + END + CLEAR + + importer = ModuleImporter.new(base_dir: Dir.pwd, use_mir: true) + expect { CompilerFrontend.compile(src, importer: importer, source_dir: Dir.pwd) } + .not_to raise_error + end +end + +RSpec.describe "lambda capture capabilities" do + # A capture is the same binding seen from inside the lambda, so it keeps the + # source's capabilities. declare_captures inherited ownership identity but not + # capabilities, so capturing a WITH alias with USE(MUTABLE ...) produced an + # entry with none -- Scope#is_restricted? was false for it, and borrowing + # through the capture (calling a `RETURNS self: T` accessor) was refused. + it "carries the source's capabilities onto a USE capture" do + src = <<~CLEAR + STRUCT Cursor { items: []Int64, pos: Int64 } + + PUB FN cursor__at(self: Cursor) RETURNS self: Int64 + REQUIRES self: LOCAL + -> + WITH POLYMORPHIC self AS view { + RETURN UNWRAP (view.items[view.pos]); + } + END + + PUB FN apply(blk: FN() -> Int64) RETURNS Int64 -> + RETURN blk(); + END + + PUB FN cursor__first(MUTABLE self: Cursor) RETURNS Int64 + REQUIRES self: LOCAL + -> + WITH POLYMORPHIC self AS MUTABLE view { + RETURN apply(%() USE(MUTABLE view) -> cursor__at(view)); + } + END + CLEAR + + importer = ModuleImporter.new(base_dir: Dir.pwd, use_mir: true) + expect { CompilerFrontend.compile(src, importer: importer, source_dir: Dir.pwd) } + .not_to raise_error + end +end diff --git a/compiler/spec/mir_emitter_spec.rb b/compiler/spec/mir_emitter_spec.rb index d5f6fa9ef..3b4b6b7f4 100644 --- a/compiler/spec/mir_emitter_spec.rb +++ b/compiler/spec/mir_emitter_spec.rb @@ -678,6 +678,28 @@ expect(e.emit(node)).to eq("User{ .id = 1, .name = \"alice\" }") end + it "drops the @as around a NIL struct field" do + # The field type may live in a package this module never imported, so + # naming it does not resolve; a struct literal infers `null` anyway. + node = MIR::StructInit.new("Node", [ + { name: "name", value: MIR::Lit.new("\"n\"") }, + { name: "plan", value: MIR::Cast.new(MIR::Lit.new("null"), "?ResourceClosePlan", :as) } + ]) + expect(e.emit(node)).to eq("Node{ .name = \"n\", .plan = null }") + end + + it "keeps a non-NIL @as in a struct field" do + node = MIR::StructInit.new("Node", [ + { name: "count", value: MIR::Cast.new(MIR::Lit.new("0"), "i64", :as) } + ]) + expect(e.emit(node)).to eq("Node{ .count = @as(i64, 0) }") + end + + it "escapes a struct field named after a Zig keyword" do + node = MIR::StructInit.new("Node", [{ name: "comptime", value: MIR::Lit.new("true") }]) + expect(e.emit(node)).to eq("Node{ .@\"comptime\" = true }") + end + it "emits anonymous struct init" do node = MIR::StructInit.new(nil, [{ name: "x", value: MIR::Lit.new("1") }]) expect(e.emit(node)).to eq(".{ .x = 1 }") diff --git a/compiler/spec/mir_gap_burn_spec.rb b/compiler/spec/mir_gap_burn_spec.rb index e21870209..997eda4ab 100644 --- a/compiler/spec/mir_gap_burn_spec.rb +++ b/compiler/spec/mir_gap_burn_spec.rb @@ -3960,7 +3960,7 @@ def malformed_array_type.element_type = raise "bad element type" nonempty_striped = AST::HashLit.new(tok, { lit("k") => lit(1, type: :Int64) }, :heap) nonempty_striped.full_type = striped_string nonempty_striped_result = hash_low.send(:lower_hash_lit, nonempty_striped) - striped_wrapped = nonempty_striped_result.body.grep(MIR::Let).find { |stmt| stmt.name == "__hm_wrapped" } + striped_wrapped = nonempty_striped_result.body.grep(MIR::Let).find { |stmt| stmt.name.start_with?("__hm_wrapped") } expect(striped_wrapped.init).to be_a(MIR::CapWrap) striped_numeric = Type.new("HashMap", ownership: :shared, sync: :locked, shard_count: 4) @@ -3981,10 +3981,10 @@ def malformed_array_type.element_type = raise "bad element type" nonempty_shared.full_type = shared_numeric nonempty_result = hash_low.send(:lower_hash_lit, nonempty_shared) expect(nonempty_result).to be_a(MIR::BlockExpr) - wrapped_let = nonempty_result.body.grep(MIR::Let).find { |stmt| stmt.name == "__hm_wrapped" } + wrapped_let = nonempty_result.body.grep(MIR::Let).find { |stmt| stmt.name.start_with?("__hm_wrapped") } expect(wrapped_let.init).to be_a(MIR::CapWrap) - expect(wrapped_let.init.inner.name).to eq("__hm") - expect(nonempty_result.body.last.value.name).to eq("__hm_wrapped") + expect(wrapped_let.init.inner.name).to start_with("__hm") + expect(nonempty_result.body.last.value.name).to start_with("__hm_wrapped") scalar_typed_list = AST::ListLit.new(tok, [], :heap) scalar_typed_list.full_type = Type.new(:Int64) diff --git a/compiler/spec/mir_lowering_spec.rb b/compiler/spec/mir_lowering_spec.rb index 9efab0fbc..99de83045 100644 --- a/compiler/spec/mir_lowering_spec.rb +++ b/compiler/spec/mir_lowering_spec.rb @@ -692,10 +692,24 @@ def collect_mir_nodes(root, klass) expect(emit(result)).to eq("push") end + it "encodes a question mark rather than dropping it" do + # `empty` and `empty?` are different CLEAR names; stripping the mark + # collapsed the pair onto one Zig identifier. + expect(emit(lowering.lower(make_id("empty?")))).to eq("empty_p") + expect(emit(lowering.lower(make_id("empty")))).to eq("empty") + end + + it "encodes a bang the same way" do + expect(emit(lowering.lower(make_id("check!")))).to eq("check_bang") + end + it "lowers identifier with question mark" do node = make_id("empty?") result = lowering.lower(node) - expect(emit(result)).to eq("empty") + # Zig carries no `?`, but CLEAR distinguishes `empty` from `empty?`. + # Stripping the mark collapsed the pair onto one Zig name -- a duplicate + # declaration, or a silent call to the wrong one -- so it is encoded. + expect(emit(result)).to eq("empty_p") end it "renames main to clearMain" do @@ -5218,3 +5232,122 @@ def typed_node(type) expect(entry.lifecycle_plan).to equal(lifecycle) end end + +RSpec.describe "block expression result type" do + # `lower_block_expr` must stamp the block's result type from its already + # annotated tail expression. Without the stamp, hoisting fell back to + # re-deriving the type from the MIR body shape, which only recognised a + # handful of break-value shapes and blew up with "allocating MIR::BlockExpr + # has no result type" on a tuple tail with more than one AllocMark. + it "stamps the tail expression's type on a multi-allocation tuple block" do + src = <<~CLEAR + FN pair(a: String, b: String) RETURNS Tuple -> + RETURN ( { MUTABLE x = COPY a; MUTABLE y = COPY b; Tuple{COPY x, COPY y} } ); + END + CLEAR + + importer = ModuleImporter.new(base_dir: Dir.pwd, use_mir: true) + result = CompilerFrontend.compile(src, importer: importer, source_dir: Dir.pwd) + fn = result.ast.statements.find { |s| s.is_a?(AST::FunctionDef) && s.name == "pair" } + low = MIRLowering.new(input: MIRLoweringInput.new( + struct_schemas: result.struct_schemas, + enum_schemas: result.enum_schemas, + union_schemas: result.union_schemas, + fn_sigs: result.fn_sigs, + lifecycle_registry: result.lifecycle_registry, + importer: importer, + source_dir: Dir.pwd, + target: :zig + )) + + mir = low.lower_program(result.ast) + blocks = [] + walk = lambda do |node| + blocks << node if node.is_a?(MIR::BlockExpr) + case node + when Array then node.each { |c| walk.call(c) } + when Struct then node.each_pair { |_, v| walk.call(v) } + end + end + walk.call(mir.items) + + block = blocks.first + expect(block).not_to be_nil, "expected the tuple tail to lower to a MIR::BlockExpr" + expect(block.result_type).not_to be_nil, + "lower_block_expr left result_type unstamped, so hoisting must re-derive it" + expect(block.result_type.resolved.to_s).to include("Tuple") + end +end + +RSpec.describe "sharded map hoisting" do + # `hoist_cleanup_entry` enumerates the allocating MIR nodes it knows how to + # clean up and raises on anything else. `MIR::ShardedMapGet` was never added + # alongside the RegistryCall/IndexedStore family it belongs to, so hoisting an + # owned sharded-map read died with "unhandled allocating MIR node". + it "derives a cleanup entry for an owned ShardedMapGet result" do + low = MIRLowering.new(input: MIRLoweringInput.new(target: :zig)) + node = MIR::ShardedMapGet.new( + MIR::Ident.new("map"), + MIR::Ident.new("key"), + nil, + nil, + :string_map, + FunctionSignature.new(params: [], return_type: Type.new(:String), intrinsic: true), + Type.new(:String), + Type.new(:String), + MIR::InlineAllocMetadata.new, + IntrinsicTemplateKind::ShardDirectZig + ) + + expect { low.send(:hoist_cleanup_entry, node, nil) }.not_to raise_error + end +end + +RSpec.describe "per-statement hoist scratch" do + # FunctionState is per-MIRLowering, not per-function, so pending_stmts (hoist + # scratch for the statement being lowered) used to survive into the NEXT + # top-level statement. Residue surfaced inside a later function's body, + # carrying AllocMark/ErrCleanup groups without the TransferMarks emitted with + # the body they belonged to -- ERRCLEANUP_WITHOUT_TRANSFER on a function that + # never allocated anything. + it "does not carry one statement's pending hoists into the next" do + src = <<~CLEAR + FN first(a: String) RETURNS String -> + RETURN COPY a; + END + + FN second(b: String) RETURNS String -> + RETURN COPY b; + END + CLEAR + + importer = ModuleImporter.new(base_dir: Dir.pwd, use_mir: true) + result = CompilerFrontend.compile(src, importer: importer, source_dir: Dir.pwd) + low = MIRLowering.new(input: MIRLoweringInput.new( + struct_schemas: result.struct_schemas, + enum_schemas: result.enum_schemas, + union_schemas: result.union_schemas, + fn_sigs: result.fn_sigs, + lifecycle_registry: result.lifecycle_registry, + importer: importer, + source_dir: Dir.pwd, + target: :zig + )) + + low.instance_variable_get(:@state).function_state.pending_stmts << MIR::Comment.new("leaked-scratch") + program = low.lower_program(result.ast) + + seen = [] + walk = lambda do |n| + case n + when Array then n.each { |c| walk.call(c) } + when MIR::Comment then seen << n.text + when Struct then n.each_pair { |_, v| walk.call(v) } + end + end + walk.call(program.items) + + expect(seen).not_to include("leaked-scratch"), + "hoist scratch from a previous statement was emitted into a later function's body" + end +end diff --git a/compiler/spec/module_scope_declaration_spec.rb b/compiler/spec/module_scope_declaration_spec.rb new file mode 100644 index 000000000..27f01073f --- /dev/null +++ b/compiler/spec/module_scope_declaration_spec.rb @@ -0,0 +1,32 @@ +require "rspec" +require_relative "../ruby/backends/transpiler" unless defined?(ZigTranspiler) + +# A container-scope declaration cannot carry a statement suffix: Zig reads +# `var x: i64 = 11; _ = &x;` at module scope as a malformed field list. The +# unused-binding suppression only belongs inside a function body. +RSpec.describe "module-scope declarations" do + it "omits the unused-binding suppression from a module-level global" do + out = ZigTranspiler.new.transpile_as_module(<<~CLEAR) + MUTABLE next_id: Int64 = 11; + + PUB FN seed() RETURNS Int64 -> + RETURN next_id; + END + CLEAR + + expect(out).to include("next_id: i64 = 11;") + expect(out).not_to include("_ = &next_id;") + end + + it "still suppresses an unused binding inside a function body" do + out = ZigTranspiler.new.transpile_as_module(<<~CLEAR) + PUB FN seed() RETURNS Int64 -> + MUTABLE buf: []Int64 = List[]; + &buf.append(1_i64); + RETURN buf.length(); + END + CLEAR + + expect(out).to include("_ = &buf;") + end +end diff --git a/compiler/spec/move_semantics_spec.rb b/compiler/spec/move_semantics_spec.rb index 4a77832e7..7c5159e4f 100644 --- a/compiler/spec/move_semantics_spec.rb +++ b/compiler/spec/move_semantics_spec.rb @@ -10,8 +10,18 @@ def transpile(src) ZigTranspiler.new.transpile(src) end + # Emitted bodies now contain nested `{ ... }` scopes whose closing brace sits + # at column 0, so stopping at the FIRST line-start `}` truncates the body. + # Take everything up to the last one before the next top-level fn instead. def fn_body(zig, name) - zig[/fn #{Regexp.escape(name)}\b.*?\n(.*?)^}/m, 1] + start = zig.index(/^(?:pub )?fn #{Regexp.escape(name)}\b/) + return nil unless start + + rest = zig[start..] + after = rest[(rest.index("\n") + 1)..] + nxt = after.index(/^(?:pub )?fn \w/) + segment = nxt ? after[0...nxt] : after + segment[/\A(.*)^}/m, 1] || segment end # ========================================================================= @@ -145,8 +155,39 @@ def fn_body(zig, name) CLEAR body = fn_body(zig, "run") expect(body).to include("defer if (!__tmp_1_moved) CheatLib.cleanup(@TypeOf(__tmp_1), __clear_heap_alloc, &__tmp_1)") - expect(body).to match(/try __hm\.put[^\n]*__tmp_1[^\n]*\n__tmp_1_moved = true;/) - expect(body).to match(/try __hm\.put[^\n]*__tmp_2[^\n]*\n__tmp_2_moved = true;/) + expect(body).to match(/try __hm_\d+\.put[^\n]*__tmp_1[^\n]*\n__tmp_1_moved = true;/) + expect(body).to match(/try __hm_\d+\.put[^\n]*__tmp_2[^\n]*\n__tmp_2_moved = true;/) + end + + it "confines each map-literal pair's cleanup guard to its own scope" do + # Zig re-emits every PENDING errdefer at each `try`, so a guard that + # stays live for the rest of the literal costs code at every later put -- + # quadratic in the entry count. One 600-entry registry literal compiled + # to 353 MB of machine code this way. The guard count live at the last + # put must therefore be bounded, not proportional to the entry count. + live_at_last_put = lambda do |entries| + pairs = entries.times.map { |i| %("k#{i}": COPY "v#{i}") }.join(", ") + body = fn_body(transpile(<<~CLEAR), "reg") + FN reg() RETURNS !{String}String -> + RETURN {#{pairs}}; + END + FN main() RETURNS Void -> + RETURN; + END + CLEAR + depth = 0 + open_guards = Hash.new(0) + body.lines.each do |line| + break if line.include?("put(") && line.include?("k#{entries - 1}") + + open_guards[depth] += 1 if line.start_with?("errdefer") + depth += line.count("{") - line.count("}") + open_guards.delete_if { |guard_depth, _| guard_depth > depth } + end + open_guards.values.sum + end + + expect(live_at_last_put.call(8)).to eq(live_at_last_put.call(2)) end end diff --git a/compiler/spec/multi_file_package_spec.rb b/compiler/spec/multi_file_package_spec.rb index 6c3e3208b..9009ef6e3 100644 --- a/compiler/spec/multi_file_package_spec.rb +++ b/compiler/spec/multi_file_package_spec.rb @@ -78,6 +78,92 @@ def pkg_flags(shapes, points) end end + it "initializes a package CONST whose initializer is a runtime call" do + Dir.mktmpdir do |dir| + rules = write(dir, "rules.clear", <<~CLEAR) + PUB STRUCT Rule { name: String } + + PUB FN build_index() RETURNS {String}Rule -> + MUTABLE index: {String}Rule = {}; + index["a"] = Rule{ name: "alpha" }; + RETURN index; + END + + PUB CONST RULE_INDEX: {String}Rule = build_index(); + + PUB FN lookup(key: String) RETURNS Bool -> + RETURN RULE_INDEX.contains?(key); + END + CLEAR + + main = write(dir, "main.clear", <<~CLEAR) + REQUIRE "pkg:rules" AS rules + + FN main() RETURNS Void -> + ASSERT lookup("a"), "package CONST is populated"; + ASSERT !lookup("zz"), "package CONST has only its own keys"; + RETURN; + END + CLEAR + + binary = File.join(dir, "main") + out = clear("build", main, "-o", binary, "--pkg", "rules=#{rules}") + expect(out).to include("Built:") + run_out, status = Open3.capture2e(binary) + expect(status.success?).to be(true), run_out + end + end + + it "retains an @multiowned argument kept by an imported package function" do + Dir.mktmpdir do |dir| + lex = write(dir, "lex.clear", <<~CLEAR) + PUB STRUCT Budget { limit: Int64 } + PUB STRUCT Lexer { budget: Budget@multiowned, tag: Int64 } + PUB STRUCT Parser { budget: Budget@multiowned, tag: Int64 } + + PUB FN make_budget() RETURNS Budget@multiowned -> + RETURN Budget{ limit: 10 } @multiowned; + END + + PUB FN lexer_new(budget: ?Budget = NIL) RETURNS !Lexer@multiowned -> + MUTABLE self = Lexer{ budget: (budget OR_ELSE make_budget()), tag: 1 }; + RETURN self @multiowned; + END + + PUB FN parser_new(budget: ?Budget = NIL) RETURNS !Parser@multiowned -> + MUTABLE self = Parser{ budget: (budget OR_ELSE make_budget()), tag: 2 }; + RETURN self @multiowned; + END + CLEAR + + main = write(dir, "main.clear", <<~CLEAR) + REQUIRE "pkg:lex" AS lex + + FN parse_source() RETURNS !Int64 -> + MUTABLE budget = make_budget(); + MUTABLE lexer = TRY (lexer_new(budget)); + MUTABLE parser = TRY (parser_new(budget)); + RETURN (lexer.budget.limit + parser.budget.limit); + END + + FN main() RETURNS !Void -> + total = TRY (parse_source()); + ASSERT total == 20, "both keepers see the budget"; + RETURN; + END + CLEAR + + binary = File.join(dir, "main") + out = clear("build", main, "-o", binary, "--pkg", "lex=#{lex}") + expect(out).to include("Built:") + # Two keepers, one handle: without a retain on the first call the second + # cleanup underflows the refcount. + run_out, status = Open3.capture2e(binary) + expect(status.success?).to be(true), run_out + expect(run_out).not_to include("integer overflow") + end + end + it "runs member TEST blocks via a pkg: root" do Dir.mktmpdir do |dir| shapes, points = fixture(dir) diff --git a/compiler/spec/package_union_schema_spec.rb b/compiler/spec/package_union_schema_spec.rb new file mode 100644 index 000000000..c678c4463 --- /dev/null +++ b/compiler/spec/package_union_schema_spec.rb @@ -0,0 +1,36 @@ +require "rspec" +require "tmpdir" +require_relative "../ruby/backends/transpiler" unless defined?(ZigTranspiler) + +# MATCH dispatch reads union_schemas to choose a switch-with-payload over a tag +# equality chain. A package REQUIRE emitted type aliases but never merged the +# imported schemas, so `Imported.Variant AS payload` in a consuming package +# lowered to `value == Imported.Variant` and left `payload` undeclared. +RSpec.describe "package union schema import" do + it "lowers a MATCH on an imported union to a payload switch" do + Dir.mktmpdir do |dir| + lib = File.join(dir, "lib.clear") + File.write(lib, <<~CLEAR) + PUB STRUCT Circle { radius: Int64 } + PUB STRUCT Square { side: Int64 } + PUB UNION Shape { Circle: Circle, Square: Square } + CLEAR + + out = ZigTranspiler.new.transpile_as_module(<<~CLEAR, source_dir: dir, pkg_paths: { "shapes" => lib }) + REQUIRE "pkg:shapes"; + + PUB FN shape_size(shape: Shape) RETURNS Int64 -> + PARTIAL MATCH shape START + Shape.Circle AS payload -> RETURN payload.radius;, + Shape.Square AS payload -> RETURN payload.side; + END + RETURN 0_i64; + END + CLEAR + + expect(out).to include("switch (shape)") + expect(out).to include(".Circle => |__match_payload_") + expect(out).not_to include("shape == Shape.Circle") + end + end +end diff --git a/compiler/spec/pipeline_backend_coverage_spec.rb b/compiler/spec/pipeline_backend_coverage_spec.rb index a417cc6e9..0ef47ead1 100644 --- a/compiler/spec/pipeline_backend_coverage_spec.rb +++ b/compiler/spec/pipeline_backend_coverage_spec.rb @@ -399,6 +399,7 @@ def initialize def services PipelineEachLowerer.new( source_alloc_fact: ->(_value, _name, _type_info) { nil }, + loop_mark_stmts: -> { [] }, bc_target: -> { @bc_target }, visit_mir: ->(node) { each_visit_mir(node) }, visit_body_with_placeholder: ->(_body_stmts, placeholder) { @@ -1334,6 +1335,21 @@ def soa_type(collection) expect(each_lowerer.lower(scalar, each_op)).to be_nil end + it "names the range capture and vouches for it" do + range = typed(AST::RangeLit.new(tok, lit(0), lit(2), true), Type.new(:"~Int64[]")) + + with_placeholder = each_lowerer.lower(range, each_op) + expect(with_placeholder.capture).to eq("__each_item") + expect(with_placeholder.iter.end_val).to eq(MIR::BinOp.new("+", MIR::Lit.new("2"), MIR::Lit.new("1"))) + + # A body that ignores the item keeps the same capture: Zig rejects an + # unused one, so the body opens with `_ = &__each_item;` rather than the + # capture being renamed on a usage guess the scan can get wrong. + each_host.use_placeholder = false + without_placeholder = each_lowerer.lower(range, AST::EachOp.new(tok, [])) + expect(without_placeholder.capture).to eq("__each_item") + expect(without_placeholder.body.first).to be_a(MIR::Suppress) + end it "lowers range literals with placeholder-aware capture names" do range = typed(AST::RangeLit.new(tok, lit(0), lit(2), true), Type.new(:"~Int64[]")) @@ -1341,12 +1357,17 @@ def soa_type(collection) expect(with_placeholder.capture).to eq("__each_item") expect(with_placeholder.iter.end_val).to eq(MIR::BinOp.new("+", MIR::Lit.new("2"), MIR::Lit.new("1"))) + # The capture is always bound and vouched for with a Suppress, rather + # than predicting from the body whether it is read: Zig rejects an unused + # capture, and `list |> EACH { count = count + 1; }` is ordinary. each_host.use_placeholder = false without_placeholder = each_lowerer.lower(range, AST::EachOp.new(tok, [])) - expect(without_placeholder.capture).to eq("_") + expect(without_placeholder.capture).to eq("__each_item") end + end + describe PipelineContextState do it "derives immutable context snapshots for pipeline placeholder state" do fields = Set[:age] @@ -2553,3 +2574,30 @@ def block_breaking_on(value, borrowed_view:) end end end + +RSpec.describe PipelinePlaceholderRewriter do + # HashLit pairs are keyed by AST nodes, and `storage` is a Struct member that + # escape analysis stamps after the key is already in the hash. That leaves the + # key's hash bucket stale, so re-looking it up with `fetch` raised KeyError on + # a key `pairs.keys` had just handed back. + it "rewrites pairs whose key node was mutated after insertion" do + tok = Lexer::Token.new(:CHAR, ":", 1, 1) + key = AST::Literal.new(tok, :SYMBOL, "sym", nil) + value = AST::Identifier.new(tok, "_") + node = AST::HashLit.new(tok, { key => value }, nil) + + key.storage = :heap + + context = PipelineContextState.new( + placeholder_name: "_", + acc_placeholder: nil, + join_param_map: nil, + named_bindings: {}, + soa_each_mode: false, + soa_rewrite_active: false, + soa_needed_fields: Set.new + ) + + expect { PipelinePlaceholderRewriter.new(context).substitute(node) }.not_to raise_error + end +end diff --git a/compiler/spec/pipeline_package_call_spec.rb b/compiler/spec/pipeline_package_call_spec.rb new file mode 100644 index 000000000..e49b0ddcf --- /dev/null +++ b/compiler/spec/pipeline_package_call_spec.rb @@ -0,0 +1,39 @@ +require "rspec" +require "tmpdir" +require_relative "../ruby/backends/transpiler" unless defined?(ZigTranspiler) + +# A cross-package call is qualified by the importing module's alias, stamped on +# the AST node. The pipeline placeholder rewriter rebuilds call nodes, and the +# metadata copy dropped that stamp -- so the same call emitted bare inside a +# pipeline body and qualified everywhere else. +RSpec.describe "cross-package calls inside a pipeline body" do + it "keeps the module alias when the rewriter rebuilds the call" do + Dir.mktmpdir do |dir| + lib = File.join(dir, "lib.clear") + File.write(lib, <<~CLEAR) + PUB STRUCT Spec { name: String } + + PUB FN describe_value(self: Spec) RETURNS String -> + RETURN COPY self.name; + END + CLEAR + + out = ZigTranspiler.new.transpile_as_module(<<~CLEAR, source_dir: dir, pkg_paths: { "helpers" => lib }) + REQUIRE "pkg:helpers"; + + PUB STRUCT Holder { specs: []Spec } + + PUB FN names(self: Holder) RETURNS ![]String + REQUIRES self: LOCAL + -> + WITH POLYMORPHIC self AS view { + RETURN view.specs |> SELECT COPY describe_value(_); + } + END + CLEAR + + expect(out).to match(/__clear_module_\w+\.describe_value\(/) + expect(out).not_to match(/[^.\w]describe_value\(rt/) + end + end +end diff --git a/compiler/spec/predicate_rewriter_spec.rb b/compiler/spec/predicate_rewriter_spec.rb index 9174f9a77..27b7fad17 100644 --- a/compiler/spec/predicate_rewriter_spec.rb +++ b/compiler/spec/predicate_rewriter_spec.rb @@ -51,6 +51,21 @@ def fmt(src) expect(rw(src)).not_to include("x != NIL") end + it "keeps the receiver when the operand is an index or field access" do + # GetIndex/GetField carry the `[` / `.` token, not the receiver's, so a + # span taken from the node's own token started mid-expression and the + # rewrite orphaned the receiver (`m[:a]` became `m([:a])`). + src = <<~CLEAR + FN main() RETURNS Void -> + m: {String@symbol}Int64 = {:a: 1}; + IF m[:a] != NIL THEN RETURN; END + RETURN; + END + CLEAR + expect(rw(src)).to include("(m[:a]).present?()") + expect(rw(src)).not_to include("m([:a])") + end + it "leaves the reversed `NIL == x` form alone (RHS-only rewrite in v1)" do # The reversed form is rare and bounding the right operand's # source span without a full expression parser is unreliable. diff --git a/compiler/spec/select_tense_matrix_spec.rb b/compiler/spec/select_tense_matrix_spec.rb index 47e8da744..f5fbf7fa2 100644 --- a/compiler/spec/select_tense_matrix_spec.rb +++ b/compiler/spec/select_tense_matrix_spec.rb @@ -168,7 +168,9 @@ def expect_selected_matches_declaration(source) expect(out).to match(/const __tmp_\d+ = try __select_promise\d+\.next\(\)/) expect(out).to match(/const __select_promise\d+ = try plainLater\(/) expect(out).to match(/const __select_promise\d+ = try later\(/) - expect(out).to match(/\(try __select_promise\d+\.next\(\)\)\.value/) + # The awaited value is unwrapped before `.value` is read, whether the + # await is read inline or through a hoisted temp. + expect(out).to match(/\(try (?:__tmp_\d+|__select_promise\d+\.next\(\))\)\.value/) expect(out).not_to include("try try") end diff --git a/compiler/spec/symbol_spec.rb b/compiler/spec/symbol_spec.rb index 44f87476f..58fb7af31 100644 --- a/compiler/spec/symbol_spec.rb +++ b/compiler/spec/symbol_spec.rb @@ -199,9 +199,11 @@ def main_body(src) expect(t.ownership_bearing?).to be false end - it "zig_type is []const u8 (same wire type as String)" do + it "zig_type is a distinct Symbol, not the String wire type" do + # Sharing []const u8 with String is what let cleanup free an interned + # handle: nothing downstream could tell the two apart. t = Type.new(:String, sync: :symbol) - expect(t.zig_type).to eq("[]const u8") + expect(t.zig_type).to eq("CheatLib.Symbol") end it "symbol type via constructor sets provenance to :rodata explicitly" do @@ -430,8 +432,8 @@ def run(src) RETURN; END CLEAR - expect(zig).to include('const __clear_symbol_0: []const u8 = "ok";') - expect(zig).to include("const x: []const u8 = __clear_symbol_0;") + expect(zig).to include('const __clear_symbol_0: CheatLib.Symbol = .{ .bytes = "ok" };') + expect(zig).to match(/const x = __clear_symbol_\d+;/) end it "deduplicates repeated static symbol literals" do @@ -445,8 +447,8 @@ def run(src) RETURN; END CLEAR - expect(zig.scan(/const __clear_symbol_\d+: \[\]const u8 = "foo";/).size).to eq(1) - expect(zig.scan(/const __clear_symbol_\d+: \[\]const u8 = "bar";/).size).to eq(1) + expect(zig.scan(/const __clear_symbol_\d+: CheatLib\.Symbol = \.\{ \.bytes = "foo" \};/).size).to eq(1) + expect(zig.scan(/const __clear_symbol_\d+: CheatLib\.Symbol = \.\{ \.bytes = "bar" \};/).size).to eq(1) end it "emits the static symbol pool for modules before exported items" do @@ -455,11 +457,42 @@ def run(src) RETURN :ok; END CLEAR - expect(zig).to include('const __clear_symbol_0: []const u8 = "ok";') + expect(zig).to include('const __clear_symbol_0: CheatLib.Symbol = .{ .bytes = "ok" };') expect(zig.index("const __clear_symbol_0")).to be < zig.index("pub fn label") end - it "emits symbol == symbol comparison as pointer+length check" do + it "widens a symbol to its bytes at every String coercion boundary" do + # A Symbol is a distinct handle type; every String-typed position must + # read `.bytes` (a borrow of interned storage) or Zig rejects the + # program. Pinned per-boundary so a regression names the site. + zig = compile_symbol_src(<<~CLEAR) + FN borrow_len(s: String) RETURNS Int64 -> + RETURN s.length(); + END + FN consume(TAKES s: String) RETURNS Int64 -> + RETURN s.length(); + END + FN main() RETURNS Void -> + MUTABLE tag: String@symbol = :alpha; + MUTABLE casted: String = CAST(tag AS String); + n = borrow_len(tag); + MUTABLE doomed: String@symbol = :alpha; + m = consume(doomed); + print("tag is ${tag}"); + RETURN; + END + CLEAR + # CAST reads the field... + expect(zig).to match(/\.bytes/) + # ...a borrowing String param gets the bytes, not the handle... + expect(zig).to match(/borrow_len\([^)]*\.bytes\)/) + # ...TAKES gets an owned COPY of the bytes, never the interned storage... + expect(zig).to match(/dupe\(u8, [^)]*\.bytes\)/) + # ...and an interpolated symbol concatenates as bytes. + expect(zig).to match(/concat\([^;]*\.bytes/) + end + + it "emits symbol == symbol comparison as an interning-agnostic equality" do zig = compile_symbol_src(<<~CLEAR) FN main() RETURNS Void -> a = :foo; @@ -468,15 +501,15 @@ def run(src) RETURN; END CLEAR - # symbolEql expands to pointer+length comparison, not CheatLib.eql - expect(zig).to include(".ptr ==") - expect(zig).to include(".len ==") - expect(zig).to include("const a: []const u8 = __clear_symbol_0;") - expect(zig).to include("const b: []const u8 = __clear_symbol_0;") - expect(zig).not_to include("CheatLib.eql") + # Pooled literals and runtime intern-table handles never share a pointer, + # so symbolEql keeps the pointer fast path inside CheatLib.eql rather + # than comparing identity alone. + expect(zig).to include("CheatLib.eql(a, b)") + expect(zig).to match(/const a = __clear_symbol_\d+;/) + expect(zig).to match(/const b = __clear_symbol_\d+;/) end - it "emits != between symbols as negated pointer check" do + it "emits != between symbols as a negated equality" do zig = compile_symbol_src(<<~CLEAR) FN main() RETURNS Void -> a = :foo; @@ -485,7 +518,7 @@ def run(src) RETURN; END CLEAR - expect(zig).to include(".ptr ==") + expect(zig).to include("!CheatLib.eql(a, b)") end it "emits ASSERT with symbol comparison" do @@ -509,7 +542,7 @@ def run(src) RETURN; END CLEAR - expect(zig).to include('const __clear_symbol_0: []const u8 = "debug";') + expect(zig).to include('const __clear_symbol_0: CheatLib.Symbol = .{ .bytes = "debug" };') expect(zig).to include("tag_label(__clear_symbol_0)") end @@ -522,9 +555,9 @@ def run(src) RETURN; END CLEAR - expect(zig).to include('const __clear_symbol_0: []const u8 = "release";') - # Return type is []const u8 (same wire type) - expect(zig).to include("[]const u8") + expect(zig).to include('const __clear_symbol_0: CheatLib.Symbol = .{ .bytes = "release" };') + # The return type is the Symbol handle, not the String wire type. + expect(zig).to match(/fn mode\(.*\) !?CheatLib\.Symbol/) end it "lowers symbol intrinsic to runtime interning" do diff --git a/compiler/spec/transpiler_spec.rb b/compiler/spec/transpiler_spec.rb index 1a095c2ad..644fbd290 100644 --- a/compiler/spec/transpiler_spec.rb +++ b/compiler/spec/transpiler_spec.rb @@ -1334,6 +1334,27 @@ def function_body(zig, name) expect(zig).to include("errdefer CheatLib.cleanup(@TypeOf(__dupe_errMsg), alloc, &__dupe_errMsg)") expect(zig).to include("result.errKind = __dupe_errKind") end + + # A switch EXPRESSION gives every arm its own result temp, and a by-value + # capture adds a copy of each payload, so the frame grows with the variant + # count. The parser's 130-variant Locatable reached 2.5 MB that way and + # faulted in the prologue on a 4 MB stack. + it "clones a union arm by arm so the frame does not scale with variant count" do + src = <<~CLEAR + STRUCT Wrapped { label: String } + UNION Shape { Left: Wrapped, Right: Wrapped } + FN main() RETURNS Void -> + v = Shape{ Left: Wrapped{ label: "x" } }; + copied = COPY v; + RETURN; + END + CLEAR + zig = transpile(src) + + expect(zig).not_to include("return switch (self)") + expect(zig).to include(".Left => |*__payload_Left|") + expect(zig).to include("return .{ .Left = try CheatLib.dupeValue(@TypeOf(__payload_Left.*), __payload_Left.*, alloc) }") + end end describe "RETURN fn(borrowed_arg) does NOT suppress borrowed arg cleanup" do diff --git a/compiler/spec/type_expression_spec.rb b/compiler/spec/type_expression_spec.rb index 52db6a655..40e563475 100644 --- a/compiler/spec/type_expression_spec.rb +++ b/compiler/spec/type_expression_spec.rb @@ -3,6 +3,8 @@ require_relative "../ruby/ast/lexer" unless defined?(Lexer) require_relative "../ruby/ast/ast" unless defined?(AST::Node) require_relative "../ruby/ast/type" unless defined?(Type) +require_relative "../ruby/ast/parser" unless defined?(ClearParser) +require_relative "../ruby/semantic/tense_operation_plan" unless defined?(TenseOperationPlanner) RSpec.describe TypeExpressionParser do def expression(source) @@ -452,3 +454,38 @@ def named(name) end end end + +RSpec.describe "tense-prefixed inline type capabilities" do + def annotation(source) + ClearParser.new(Lexer.new(source).tokenize, source).send(:parse_type_annotation) + end + + # `?[]T` must carry the same `collection: :list` capability as `[]T`; the + # inline prefix path used to drop it, so the lifecycle inventory keyed the + # declared field type differently from the sink type at construction sites. + it "keeps the wrapped type's collection capability across ?, ! and ~ prefixes" do + expect(annotation("[]Int64").collection).to eq(:list) + + ["?", "!", "~"].each do |prefix| + expect(annotation("#{prefix}[]Int64").collection).to eq(:list), + "#{prefix}[]Int64 dropped the wrapped list capability" + end + end +end + +RSpec.describe "OR_ELSE result capabilities" do + # OR_ELSE consumes tense layers, not capabilities. The payload expression + # does not carry the source type's sync/collection/ownership, so rebuilding + # the result from it alone silently turned `?String@symbol` into a plain + # String -- a mismatch that reported as "Cannot assign String to String". + it "keeps the source capabilities on the recovered payload type" do + source = ClearParser.new(Lexer.new("?String@symbol").tokenize, "?String@symbol") + .send(:parse_type_annotation) + expect(source.sync).to eq(:symbol) + + plan = TenseOperationPlanner.or_else(source, Type.new(:String)) + expect(plan.result_type.sync).to eq(:symbol), + "OR_ELSE dropped @symbol from the recovered payload" + end + +end diff --git a/compiler/spec/type_zig_type_gap_spec.rb b/compiler/spec/type_zig_type_gap_spec.rb index 1715fd981..248db8d40 100644 --- a/compiler/spec/type_zig_type_gap_spec.rb +++ b/compiler/spec/type_zig_type_gap_spec.rb @@ -192,8 +192,9 @@ expect(heap.placement.location).to eq(:heap) expect(heap.location).to eq(:heap) expect(fallback.apply_cleanup_placement!(value_type: nil, alloc: nil)).to equal(fallback.placement) - expect(Type.new(:"Int64[]").dynamic_field_array?).to be true - expect(Type.new(:"Int64[2]", collection: :list).dynamic_field_array?).to be true + # `Int64[]` in a field is a Zig slice; `[]Int64@list` is an ArrayList. + expect(Type.new(:"Int64[]").slice_shaped_field_array?).to be true + expect(Type.new(:"Int64[2]", collection: :list).slice_shaped_field_array?).to be false end it "applies element-level capabilities to array element types" do diff --git a/compiler/spec/with_view_codegen_spec.rb b/compiler/spec/with_view_codegen_spec.rb index a82bc3763..7ec5e8451 100644 --- a/compiler/spec/with_view_codegen_spec.rb +++ b/compiler/spec/with_view_codegen_spec.rb @@ -11,8 +11,18 @@ def transpile(src) ZigTranspiler.new.transpile(src) end + # Emitted bodies now contain nested `{ ... }` scopes whose closing brace sits + # at column 0, so stopping at the FIRST line-start `}` truncates the body. + # Take everything up to the last one before the next top-level fn instead. def fn_body(zig, name) - zig[/fn #{name}\b.*?\n(.*?)^\}/m, 1] || "" + start = zig.index(/^(?:pub )?fn #{Regexp.escape(name)}\b/) + return "" unless start + + rest = zig[start..] + after = rest[(rest.index("\n") + 1)..] + nxt = after.index(/^(?:pub )?fn \w/) + segment = nxt ? after[0...nxt] : after + segment[/\A(.*)^}/m, 1] || segment || "" end describe "Phase 2.5 — scalar WITH VIEW" do diff --git a/sorbet/config b/sorbet/config index 2961dc561..41a6c9a2a 100644 --- a/sorbet/config +++ b/sorbet/config @@ -55,3 +55,4 @@ --suppress-error-code=7034 --suppress-error-code=7006 --suppress-error-code=7050 +--ignore=compiler/.ruby-rbs/ diff --git a/tools/fuzz/README.md b/tools/fuzz/README.md index b4f56cd54..91ee60cd0 100644 --- a/tools/fuzz/README.md +++ b/tools/fuzz/README.md @@ -56,7 +56,7 @@ actually selected, rejects both baseline and semantic runs if any result is a timeout, compares individual surviving mutant IDs, and writes `semantic-mutant-delta/v1` facts. - bundle exec ruby gems/gigasail/tools/mutant-converters/semantic_mutant.rb \ + bundle exec ruby gems/lineage/tools/mutant-converters/semantic_mutant.rb \ --out /tmp/clear-semantic-mutants --timeout 60 --min-new-kills 1 The final paired run selected the same 369 parser mutants on both sides and @@ -211,7 +211,6 @@ expected hard error is absent. | `fsm_edge_matrix` | 8 | Additional FSM splitter edges around OR fallbacks, nested loop/branch suspension, stream branches, locks before NEXT, and known early-return lowering failures. | | `diagnostic_policy_matrix` | 16 | Policy-heavy front-end diagnostics for reentrancy, hold-lock-across-yield, lock ordering, handlers, and ownership/fixable rejection paths. | | `pipeline_source_shape_matrix` | 44 | Pipeline source/terminal shapes across range, BG STREAM, bounded promises, strings, and observable terminals. | -| `pipeline_consumer_position_matrix` | 26 | Where a pipeline result lands: bound inside FOR/WHILE/MATCH-arm-in-loop, iterated under IF, used as an IF condition, re-piped mid-chain, or escaped into TAKES, a struct field, or an outer list. | | `semantic_equivalence_matrix` | 531 | Recursively derived Int64, Bool, String, struct, list, map, and Tuple equivalences crossed with compatible local, call, aggregate, ownership, and pipeline slots — including stream-pipeline productions (identity SELECT into fused SUM, observing selectors over owned stream items, identity re-stream drained by WHILE-EXISTS). | | `semantic_gap_matrix` | 21 | Raw positive witnesses for every fixed compiler defect found by the original, capability-expansion, whole-program, and migration-completion campaigns. | | `semantic_capability_matrix` | 17 | Closed reviewed capability allowlist across String, struct, list, map, Tuple, synchronized struct, and shared-atomic Int64 payloads. | @@ -238,7 +237,9 @@ expected hard error is absent. | `extern_boundary_matrix` | 6 | Negative extern declaration/call boundaries for free functions, trampolines, extern methods/resources, generic comptime calls, and tight-loop rejection. | | `kept_identity_matrix` | 105 | Retained identity v4 keep edges: caller model x destination x post-call use x arity x fallibility; declaration-sited negative cells (KEPT_IDENTITY_NEEDS_MODEL, use-after-GIVE). | | `carrier_ownership_matrix` | 15 | Retained identity v5 carrier ownership: source carrier x contract x fan-out. Positives leak-checked (@multiowned/@shared KEEP retain, shared->unique OWN COPY, last-use move, SHARED multi-consume, OWN COPY detach, MONOMORPHIC carrier threading, MONOMORPHIC KEEP per carrier); negatives pin KEEP_ON_KNOWN_CARRIER, COPY_ON_POLYMORPHIC_PARAM, COPY_RETAINED_NEEDS_UNIQUE, CARRIER_POLYMORPHIC_FANOUT, ARG_NEEDS_SHARED, RETAINED_NEEDS_OWN_COPY, OWN_ALONE_UNSUPPORTED. | -| `curated_gap_corpus` | 556 | Self-contained `transpile-tests/*.clear` corpus reused as broad compile-mode fuzz coverage for parser, annotator, MIR lowering, and emission. | +| `provenance_round_trip_matrix` | 36 | A value read back out of a map, list, struct field, or optional keeps the provenance it was stored with: owned values are freed exactly once, statics and borrows never. | +| `pipeline_consumer_position_matrix` | 26 | Where a pipeline result lands: bound inside FOR/WHILE/MATCH-arm-in-loop, iterated under IF, used as an IF condition, re-piped mid-chain, or escaped into TAKES, a struct field, or an outer list. | +| `curated_gap_corpus` | 605 | Self-contained `transpile-tests/*.clear` corpus reused as broad compile-mode fuzz coverage for parser, annotator, MIR lowering, and emission. | | `tense_predicate_matrix` | 11 | Postfix tense predicates, stacked refinement, readiness polling, and ambiguous optional-Boolean rejection. | | `next_tense_matrix` | 9 | NEXT across future/stream values and their fallible/optional tense permutations, including invalid redundant and missing unwraps. | | `tense_operation_plan_matrix` | 34 | Executable annotation-to-MIR handoff coverage for TRY, UNWRAP, OR_ELSE, tense predicates, ordered tense navigation, scalar NEXT, and fallible promise-list aggregation. | diff --git a/tools/fuzz/coverage_model.rb b/tools/fuzz/coverage_model.rb index bcfae17e1..393b06c85 100644 --- a/tools/fuzz/coverage_model.rb +++ b/tools/fuzz/coverage_model.rb @@ -268,6 +268,12 @@ def self.profile(failure_proves:, high_risk: false, known_exclusions: [], matrix failure_proves: 'Rc/Arc operations recursively retain and finalize String-owning payloads through every generic operation.', high_risk: true ), + provenance_round_trip_matrix: profile( + failure_proves: 'A value read back out of a container, struct, or optional keeps the ' \ + 'provenance it was stored with: owned values are freed exactly once, ' \ + 'and statics are never freed.', + high_risk: true + ), match_matrix: profile( failure_proves: 'MATCH lowering over union/scalar shapes binds payloads and cleans owned arms.' ), diff --git a/tools/fuzz/mutants/patches/symbol_spelled_as_string.patch b/tools/fuzz/mutants/patches/symbol_spelled_as_string.patch new file mode 100644 index 000000000..99d7a2ac4 --- /dev/null +++ b/tools/fuzz/mutants/patches/symbol_spelled_as_string.patch @@ -0,0 +1,18 @@ +diff --git a/compiler/ruby/ast/type.rb b/compiler/ruby/ast/type.rb +index 5fdcd05605..8eb2f2662a 100644 +--- a/compiler/ruby/ast/type.rb ++++ b/compiler/ruby/ast/type.rb +@@ -5676,11 +5676,8 @@ class Type + return signed_integer? ? "isize" : "usize" + end + if resolved == :String || string? +- # An interned symbol is represented exactly like a String and owned by +- # nobody. Spelling both []const u8 left every downstream consumer -- +- # cleanup above all -- unable to tell them apart. +- return "CheatLib.Symbol" if symbol? +- ++ # MUTANT: spell a symbol the same as a String, so nothing downstream can ++ # tell an interned handle from an owned one. + return "[]const u8" + end + diff --git a/tools/fuzz/mutants/registry.rb b/tools/fuzz/mutants/registry.rb index 9156340ba..832210810 100644 --- a/tools/fuzz/mutants/registry.rb +++ b/tools/fuzz/mutants/registry.rb @@ -24,6 +24,16 @@ class Mutant < T::Struct PATCH_DIR = T.let(File.expand_path('patches', __dir__), String) REGISTRY = T.let([ + Mutant.new( + name: :symbol_spelled_as_string, + description: 'Render `String@symbol` as []const u8 again, so an interned handle is ' \ + 'indistinguishable from an owned String and a container of symbols frees ' \ + 'the .rodata behind them.', + invariant: :symbol_provenance_round_trip, + patch: File.join(PATCH_DIR, 'symbol_spelled_as_string.patch'), + templates: [:provenance_round_trip_matrix], + kill: { bucket: :fail, min_delta: 1 } + ), Mutant.new( name: :pipeline_reduce_owned_accumulator_unclassified, description: 'Stop descending value-BlockExpr bodies during cleanup classification, so a desugared REDUCE\'s owned (String) accumulator never gets a cleanup entry and its per-step reassignment falls back to a bare Set. The composite-element matrix must reject the resulting unhoisted/leaked owned accumulator.', diff --git a/tools/fuzz/run.rb b/tools/fuzz/run.rb index b38dbeae6..39e006022 100755 --- a/tools/fuzz/run.rb +++ b/tools/fuzz/run.rb @@ -54,6 +54,10 @@ o.on('--clean') { opts[:clean] = true } o.on('--templates LIST') { |v| opts[:templates] = v.split(',').map(&:to_sym) } o.on('--jobs N', Integer) { |v| opts[:jobs] = v } + # Run the cells through LLVM with safety on instead of the self-hosted + # backend. Catches miscompiles the default backend introduces (the lexer + # keyword comparison was one) and safety checks a Debug arena hides. + o.on('--safe') { opts[:safe] = true } o.on('--bisect-positives') { opts[:bisect_positives] = true } o.on('--shard I/N') do |v| idx, total = v.split('/', 2).map(&:to_i) @@ -136,7 +140,7 @@ def ensure_symlink(link_path, target_path) File.symlink(target_path, link_path) end -def run_pass_bundle(entries, out_dir, bundle_name: 'all-fuzz') +def run_pass_bundle(entries, out_dir, bundle_name: 'all-fuzz', safe: false) return [[], [], [], []] if entries.empty? started = Process.clock_gettime(Process::CLOCK_MONOTONIC) @@ -187,6 +191,9 @@ def run_pass_bundle(entries, out_dir, bundle_name: 'all-fuzz') 'runtime/switch.S', 'runtime/onRoot.S', '-lc' ] + # --safe routes the bundle through LLVM with safety on rather than the + # self-hosted backend. + zig_args += ['-O', 'ReleaseSafe'] if safe out, status = if coverage_enabled ZigCoverageSupport.run_zig_test( @@ -203,15 +210,17 @@ def run_pass_bundle(entries, out_dir, bundle_name: 'all-fuzz') suffix = coverage_enabled ? " under kcov" : "" puts "[fuzz] pass bundle #{bundle_name}#{suffix}: #{entries.size} cells in #{format('%.2f', elapsed)}s" - if !status.success? || out.include?('FAIL') - return [[], [[zig_path, out]], [], []] - end - leak = out =~ /MEMORY LEAKS:\s*[1-9]/ || out.include?('[DebugAllocator] (err)') || out.include?('[gpa] (err)') || out =~ /\d+ tests leaked memory/ + + # Order matters, and it is the order the isolated lanes already use: a + # leaking bundle exits non-zero, so checking the status first classifies + # every leak as a plain failure and the leak lane never sees one. + return [[], [[zig_path, out]], [], []] if out.include?('FAIL') return [[], [], [], [[zig_path, out]]] if leak + return [[], [[zig_path, out]], [], []] unless status.success? [entries.map { |e| e[:path] }, [], [], []] ensure @@ -222,7 +231,7 @@ def run_pass_bundle(entries, out_dir, bundle_name: 'all-fuzz') end end -def run_parallel_pass_bundles(entries, out_dir, default_workers) +def run_parallel_pass_bundles(entries, out_dir, default_workers, safe: false) return [[], [], [], []] if entries.empty? started = Process.clock_gettime(Process::CLOCK_MONOTONIC) @@ -238,7 +247,7 @@ def run_parallel_pass_bundles(entries, out_dir, default_workers) pid = Process.fork do reader.close simplecov_child_command!("fuzz-pass-bundle-#{index}") - result = run_pass_bundle(chunk, out_dir, bundle_name: "all-fuzz-#{index}") + result = run_pass_bundle(chunk, out_dir, bundle_name: "all-fuzz-#{index}", safe: safe) writer.write(Marshal.dump(result)) writer.close exit 0 @@ -274,7 +283,10 @@ def print_failure_excerpt(out) first = [failure_index - 8, 0].max lines[first, 40] else - lines.first(40) + # No marker: the run died without reporting one (a panic, a signal, a + # killed test binary). Whatever happened is at the END of the output -- + # printing the first 40 lines shows 40 passing cells and nothing else. + lines.last(40) end excerpt.each { |line| puts " #{line}" } return unless lines.size > excerpt.size @@ -475,13 +487,13 @@ def run_compile_only_negative_coverage(entries, default_workers) [pass, mismatched] end -def coverage_run(emitted, out_dir, default_workers) +def coverage_run(emitted, out_dir, default_workers, safe: false) pass_entries = emitted.select { |e| e[:kind] != :mir_checker && e[:expected] == :pass } negative_entries = emitted.select { |e| e[:kind] != :mir_checker && e[:expected] == :compile_error } mir_negative_entries = emitted.select { |e| e[:kind] == :mir_checker && e[:expected] == :compile_error } if ZigCoverageSupport.enabled? - pass_ok, fails, mir_errors, leaks = run_parallel_pass_bundles(pass_entries, out_dir, default_workers) + pass_ok, fails, mir_errors, leaks = run_parallel_pass_bundles(pass_entries, out_dir, default_workers, safe: safe) else pass_ok, mir_errors, leaks = run_compile_only_positive_coverage(pass_entries, default_workers) fails = [] @@ -568,7 +580,7 @@ def run_fail_complete_bundles(entries, out_dir) result = FuzzFailComplete.run(entries) do |batch| attempts += 1 puts "[fuzz] fail-complete bundle attempt #{attempts}: #{batch.size} cells" - batch_result = run_pass_bundle(batch, out_dir) + batch_result = run_pass_bundle(batch, out_dir, safe: safe) if batch.size == 1 # A singleton bundle diagnostic belongs to its sole source cell. Keep # that identity instead of reporting the transient all-fuzz.zig path. @@ -586,7 +598,7 @@ def run_fail_complete_bundles(entries, out_dir) result end -def hybrid_run(emitted, out_dir, default_workers, bisect_positives: false) +def hybrid_run(emitted, out_dir, default_workers, bisect_positives: false, safe: false) pass_entries = emitted.select { |e| e[:kind] != :mir_checker && e[:expected] == :pass } negative_entries = emitted.select { |e| e[:kind] != :mir_checker && e[:expected] == :compile_error } mir_negative_entries = emitted.select { |e| e[:kind] == :mir_checker && e[:expected] == :compile_error } @@ -596,7 +608,7 @@ def hybrid_run(emitted, out_dir, default_workers, bisect_positives: false) if bisect_positives run_fail_complete_bundles(bundled_pass_entries, out_dir) else - run_parallel_pass_bundles(bundled_pass_entries, out_dir, default_workers) + run_parallel_pass_bundles(bundled_pass_entries, out_dir, default_workers, safe: safe) end iso_ok, iso_fails, iso_mir_errors, iso_leaks = run_positive_files(isolated_pass_entries, out_dir, default_workers) negative_ok, unexpected_pass = run_negative_builds(negative_entries, out_dir, default_workers) @@ -619,7 +631,7 @@ def per_file_run(emitted) path, expected = entry[:path], entry[:expected] short = File.basename(path) print "[#{i + 1}/#{emitted.size}] #{short} (#{expected})... " - out = `#{clear} test #{path} 2>&1` + out = `#{clear} test #{path}#{opts[:safe] ? ' --safe' : ''} 2>&1` status = $?.exitstatus compile_error = out.include?('MIR ownership verification failed') || @@ -665,9 +677,9 @@ def per_file_run(emitted) pass, fails, leaks, mir_errors, unexpected_pass = if ENV['COVERAGE'] == '1' - coverage_run(emitted, opts[:out], opts[:jobs]) + coverage_run(emitted, opts[:out], opts[:jobs], safe: opts[:safe]) else - hybrid_run(emitted, opts[:out], opts[:jobs], bisect_positives: opts[:bisect_positives]) + hybrid_run(emitted, opts[:out], opts[:jobs], bisect_positives: opts[:bisect_positives], safe: opts[:safe]) end if ZigCoverageSupport.enabled? diff --git a/tools/fuzz/templates/provenance_round_trip_matrix.rb b/tools/fuzz/templates/provenance_round_trip_matrix.rb new file mode 100644 index 000000000..4141fe403 --- /dev/null +++ b/tools/fuzz/templates/provenance_round_trip_matrix.rb @@ -0,0 +1,125 @@ +# Template: provenance round-trip matrix. +# +# Every value in CLEAR lowers to the same Zig representation regardless of who +# owns it -- an owned String, a `@rodata` literal, a `String@symbol`, and a +# borrow into a container are all []const u8. Ownership is therefore a fact the +# compiler must CARRY, and the recurring failure is a site that re-derives it +# from expression shape and gets it wrong: a view given an owning cleanup, so +# storage someone else still owns is freed. +# +# Six of the eleven bugs the self-hosting effort surfaced were that one +# mistake wearing different hats -- a map read, a rodata list, a symbol union +# payload, an optional unwrap, a struct field, an extern borrow. Nothing in the +# corpus read a non-Copy value back OUT of a container and let it drop, which is +# the shape they all share. +# +# Axes: +# provenance -- what the value actually is, and therefore who may free it; +# container -- what it is read back out of; +# exit -- how it leaves, since each exit picks a different lowering path +# (plain bind, returned through a frame boundary, read twice so +# the second read sees whatever the first one left behind). +# +# A cell that frees a static or a borrow shows up as an arena free check panic, +# a double free, or a leak; one that drops a cleanup it owed shows up as a leak. + +# `symbol` is an axis again: a `String@symbol` is now CheatLib.Symbol, a +# distinct type whose drop is a no-op, so "is this element the container's to +# free" has one answer everywhere instead of three. +PROVENANCE_ROUND_TRIP_CELLS = [] +%i[owned rodata symbol].each do |provenance| + %i[map list struct_field optional].each do |container| + %i[bind_drop return_it reread].each do |exit_shape| + PROVENANCE_ROUND_TRIP_CELLS << { + provenance: provenance, + container: container, + exit: exit_shape, + } + end + end +end + +FuzzGenerator.register(:provenance_round_trip_matrix, cells: PROVENANCE_ROUND_TRIP_CELLS) do |p| + # The element type and the expression that produces one. `@symbol` and a bare + # literal are static: freeing either is invalid. `COPY` makes an owned heap + # string that MUST be freed exactly once. + elem_type, make_value, expected = case p[:provenance] + when :owned then ["String", 'COPY "alpha"', '"alpha"'] + # A bare literal IS the rodata case: same `String` type as the owned one, no + # COPY, so nothing was allocated and nothing may be freed. The provenance is + # the value's history, not a spelling on the type. + when :rodata then ["String", '"alpha"', '"alpha"'] + when :symbol then ["String@symbol", ':alpha', ':alpha'] + end + + # Build the container and the expression that reads one value back out. + setup, read_expr, read_again = case p[:container] + when :map + ["MUTABLE holder: {String}#{elem_type} = {};\n holder[\"k\"] = #{make_value};", + 'holder["k"]', 'holder["k"]'] + when :list + ["MUTABLE holder: []#{elem_type} = [];\n &holder.append(#{make_value});", + "holder[0]", "holder[0]"] + when :struct_field + ["MUTABLE holder = Wrapper{ slot: #{make_value} };", + "holder.slot", "holder.slot"] + when :optional + ["MUTABLE holder: ?#{elem_type} = #{make_value};", + "holder", "holder"] + end + + wrapper_def = p[:container] == :struct_field ? "STRUCT Wrapper { slot: #{elem_type} }\n\n" : "" + + # An optional container yields `?T` from every read; the others yield `?T` + # only for map/list indexing. A struct field is always present. + optional_read = %i[map list optional].include?(p[:container]) + bind = optional_read ? "UNWRAP (#{read_expr})" : read_expr + bind_again = optional_read ? "UNWRAP (#{read_again})" : read_again + + body = case p[:exit] + when :bind_drop + # Bind it and let the binding go out of scope. If the read handed back a + # view and the binding claimed ownership, this frees the container's value. + <<~BODY.chomp + MUTABLE seen: #{elem_type} = #{bind}; + ASSERT seen == #{expected}, "round-tripped value is intact"; + BODY + when :return_it + # Out through a frame boundary: a frame-allocated view cannot escape, and a + # borrow returned as owned dangles once the frame rewinds. + <<~BODY.chomp + MUTABLE seen: #{elem_type} = escape(); + ASSERT seen == #{expected}, "returned value survives its frame"; + BODY + when :reread + # Read twice. The first read is what frees a container-owned value; the + # second is what observes the damage -- exactly how the const rule-index + # bug presented, where one lookup poisoned every later one. + <<~BODY.chomp + MUTABLE first: #{elem_type} = #{bind}; + ASSERT first == #{expected}, "first read is intact"; + MUTABLE second: #{elem_type} = #{bind_again}; + ASSERT second == #{expected}, "second read sees what the first left"; + BODY + end + + if p[:exit] == :return_it + <<~CHT + #{wrapper_def}FN escape() RETURNS #{elem_type} -> + #{setup} + RETURN #{bind}; + END + + FN main() RETURNS Void -> + #{body} + END + CHT + else + <<~CHT + #{wrapper_def}FN main() RETURNS Void -> + #{setup} + #{body} + END + CHT + end +end diff --git a/tools/parser_compat.rb b/tools/parser_compat.rb index 694368168..25a513e03 100644 --- a/tools/parser_compat.rb +++ b/tools/parser_compat.rb @@ -7,6 +7,7 @@ end require 'fileutils' +require 'open3' require 'json' require 'msgpack' require 'optparse' @@ -179,6 +180,11 @@ def canonical_encode(value) pairs = value.map { |key, item| [canonical_encode(key), canonical_encode(item)] } pairs.sort_by! { |key, item| key + item } "H#{pairs.length}[#{pairs.flatten.join}]" + when Type + # Type memoizes derived state into ivars on demand, so encoding it by + # instance_variables makes the bytes depend on which accessors happened + # to run. Its resolved form is the stable identity both sides agree on. + canonical_object('Type', { 'resolved' => value.resolved }) when T::Enum canonical_object(value.class.name.split('::').last, { 'value' => value.serialize }) else @@ -198,6 +204,10 @@ def canonical_ruby_object(value) end raise "unsupported parser value: #{value.class}" if fields.empty? + # Annotator stamps are not parser output. The CLEAR encoder omits them by + # construction; a Ruby Struct always carries its members, so drop them here + # too or the two sides disagree on a field neither parser populates. + fields = fields.reject { |field, _| STAMP_FIELDS.include?(field) } canonical_object(value.class.name.split('::').last, fields) end @@ -220,7 +230,12 @@ def float_text(value) end def run_clear_payload(cases, options) - Dir.mktmpdir('parser-compat-', options[:out_dir]) do |dir| + # A fresh mktmpdir per run gave the emitted Zig a new path every time, so + # Zig's own cache never hit and every run paid the full link (tens of + # minutes). One stable directory lets an unchanged harness reuse it. + dir = File.join(options[:out_dir], 'build') + FileUtils.mkdir_p(dir) + begin source = File.join(dir, 'parser_compat.clear') binary = File.join(dir, 'parser_compat') File.write(source, clear_harness_source(cases, options[:generated_root])) @@ -234,12 +249,26 @@ def run_clear_payload(cases, options) LexerHarnessSupport::CLEAR, 'build', source, '-o', binary, '--no-stack-check', + # Recursive descent over a union whose clone frame is megabytes: the + # 64 KB debug default faults in the prologue. + '--main-tier', 'service', + # Zig's self-hosted x86_64 backend miscompiles the lexer's keyword + # comparison after the first parse in a process, so `END` lexes as a + # TYPE_ID and every parse but the first fails. Building through LLVM + # is correct; drop this once the default backend is fixed. + *ENV.fetch('PARSER_COMPAT_BUILD_FLAGS', '--safe').split, # '--force' removed: it defeated incremental compilation on every build *package_flags(options[:generated_root]), env: env ) - stdout, stderr = LexerHarnessSupport.run!(binary) + # A case that crashes the process (rather than raising) still produced + # output for every case before it. Report those instead of losing the + # whole run to one bad case. + stdout, stderr, status = Open3.capture3(binary) stdout = stderr if stdout.empty? + unless status.success? + warn "parser_compat: CLEAR exited #{status.exitstatus || status.termsig}; reporting the cases it completed" + end if options[:keep] FileUtils.cp(source, File.join(options[:out_dir], 'parser_compat.clear')) @@ -251,6 +280,8 @@ def run_clear_payload(cases, options) 'implementation' => 'clear', 'cases' => parse_clear_output(stdout) } + ensure + FileUtils.rm_rf(dir) unless options[:keep] || ENV['PARSER_COMPAT_REUSE_BUILD'] end end @@ -365,10 +396,11 @@ def strongly_connected_components(graph) end def package_flags(generated_root) + groups = package_groups(generated_root) generated = generated_relatives(generated_root).flat_map do |relative| ['--pkg', "#{package_name(relative)}=#{File.join(generated_root, relative)}"] end - grouped = package_groups(generated_root).flat_map do |name, members| + grouped = groups.flat_map do |name, members| spec = members.map { |rel| File.join(generated_root, rel) }.join(',') ['--pkg', "#{name}=#{spec}"] end @@ -380,6 +412,19 @@ def package_flags(generated_root) # The harness must enter the parser through whatever package actually owns # it: its SCC group when it is cyclic, otherwise the file itself. + # Type lives in a different SCC group than the parser, so the harness has to + # require it explicitly to call type__resolved. + def type_require_spec(generated_root) + group = package_groups(generated_root).find { |_name, members| members.include?('ast/type.clear') } + group ? "pkg:#{group.first}" : File.join(generated_root, 'ast', 'type.clear') + end + + # The lexer is its own package; the harness tokenizes before parsing. + def lexer_require_spec(generated_root) + group = package_groups(generated_root).find { |_name, members| members.include?('ast/lexer.clear') } + group ? "pkg:#{group.first}" : "pkg:#{package_name('ast/lexer.clear')}" + end + def parser_require_spec(generated_root) group = package_groups(generated_root).find { |_name, members| members.include?('ast/parser.clear') } return "pkg:#{group.first}" if group @@ -387,20 +432,409 @@ def parser_require_spec(generated_root) File.join(generated_root, 'ast', 'parser.clear') end + # --- generated node encoders ------------------------------------------- + # + # The old hand-written encodeCompat() walked values with Ruby-style + # reflection (`object.class().members()`), which CLEAR does not have, so it + # never compiled. Instead, emit one encoder per node type: Ruby's Struct + # members define WHICH fields are encoded (matching canonical_ruby_object) + # and the CLEAR declarations define HOW each is encoded. + + STAMP_FIELDS = %w[ + can_fail coerced_type_object collection_return container_borrow error_kind + error_type implicit_layout_cost kept_edge_plan kept_edge_plans + layout_transport matched_signature matched_stdlib_def mutates_receiver + needs_heap_create needs_mut_ref resource_close_plan slot_size source_range + stdlib_allocates storage_override tense_plan type_object var_mutated + var_used was_moved zig_pattern symbol generic_params + ].freeze + + def clear_struct_fields(generated_root) + @clear_struct_fields ||= begin + table = {} + Dir[File.join(generated_root, 'ast', '**', '*.clear')].each do |path| + File.read(path).scan(/^(?:PUB )?STRUCT (\w+) \{(.*?)\n\}/m) do |name, body| + # A field type can hold commas (`[]Tuple`), so match to end of + # line and drop the trailing separator instead of stopping at `,`. + table[name] = body.scan(/^\s*(\w+):\s*(.+?),?\s*$/).to_h + end + end + table + end + end + + # Union types the translated sources declare, as {name => [[variant, payload]]}. + # A union encodes as its ACTIVE variant's payload, which is exactly what the + # Ruby side wrote before the translation gave the slot a name. + def clear_union_variants(generated_root) + @clear_union_variants ||= begin + table = {} + # ast/ and the parser are the translation under test; a same-named union + # elsewhere (mir/, semantic/) is a different type and would collide with + # the struct encoder of that name. + Dir[File.join(generated_root, 'ast', '**', '*.clear')].each do |path| + File.read(path).scan(/^(?:PUB )?UNION (\w+) \{(.+?)\}$/) do |name, body| + table[name] = body.split(',').filter_map do |pair| + variant, payload = pair.split(':', 2).map(&:strip) + [variant, payload] if variant && payload && !variant.empty? + end + end + end + table + end + end + + # Node classes actually produced by the corpus. Anything outside this set + # gets a loud panic rather than a silently wrong encoding. + # AST nodes are a mix of Ruby Structs and T::Structs. + def struct_member_names(klass) + klass.respond_to?(:members) ? klass.members.map(&:to_s) : klass.props.keys.map(&:to_s) + end + + def corpus_node_classes(cases) + seen = {} + @never_populated = Hash.new { |h, k| h[k] = {} } + walk = lambda do |value, guard| + return if value.nil? || guard.include?(value.object_id) + guard << value.object_id + case value + when Array then value.each { |item| walk.call(item, guard) } + when Hash then value.each { |k, v| walk.call(k, guard); walk.call(v, guard) } + when Lexer::Token then nil + when Struct + name = value.class.name.split('::').last + seen[name] = value.class + value.members.each do |m| + @never_populated[name][m.to_s] = @never_populated[name].fetch(m.to_s, true) && value[m].nil? + walk.call(value[m], guard) + end + when Type, T::Enum + # Encoded by identity, not by walking their fields. + nil + when T::Struct + # AST nodes are not all plain Structs -- EffectSpan is a T::Struct, and + # missing it here marked the class dead, so the generated encoder was a + # panic stub that fired the moment a case populated it. + name = value.class.name.split('::').last + seen[name] = value.class + value.class.props.keys.each do |m| + member = value.public_send(m) + @never_populated[name][m.to_s] = @never_populated[name].fetch(m.to_s, true) && member.nil? + walk.call(member, guard) + end + end + end + cases.each do |entry| + ast = ClearParser.new(Lexer.new(entry['source']).tokenize, entry['source']).parse + walk.call(ast, Set.new) + end + seen + end + + def clear_base_type(type) + type.to_s.sub(/@[\w:]+(\(\d+\))?/, '').strip + end + + # Returns [prelude_statements, expression] encoding `expr`, which has + # declared type `type`. Collections need a loop, so they emit a prelude. + def clear_value_encoder(type, expr, fields, slot = 'v0') + bare = clear_base_type(type) + # A @boxed field holds its value indirectly (the AST's recursive edges are + # boxed so Zig can size the types). Copy the pointee out before encoding -- + # the wire format describes the value, not the indirection. + # Only when the indirection is on THIS value, not nested inside a + # collection element -- `{String}T@multiowned` must recurse to the element, + # not copy the whole map. + if type.to_s =~ /\A\??[A-Za-z_][\w<>, ]*@boxed\z/ + return clear_value_encoder(bare, "COPY #{expr}", fields, slot) + end + # A retained handle needs OWN COPY: plain COPY is a memcpy and is illegal + # on a live @multiowned value. + if type.to_s =~ /\A\??[A-Za-z_][\w<>, ]*@multiowned\z/ + return clear_value_encoder(bare, "OWN COPY #{expr}", fields, slot) + end + + # encodeTokenValue already takes the optional -- it is how Token#value is + # encoded on both sides -- so do not unwrap it first. + return ['', "encodeTokenValue(#{expr})"] if bare == '?TokenValue' + + if bare.start_with?('?') + # Recurse on the ORIGINAL type minus the `?`, not on `bare`: clear_base_type + # has already dropped the capability, and `?String@symbol` must still encode + # as a symbol once unwrapped. + pre, inner = clear_value_encoder(type.to_s.sub(/\A\?/, ''), "#{slot}_some", fields, "#{slot}i") + body = pre.empty? ? "" : pre + "\n" + return [ + " MUTABLE #{slot} = \"N\";\n IF #{expr} EXISTS AS #{slot}_some THEN\n#{body} #{slot} = #{inner};\n END", + slot + ] + end + + # Ruby keys HashLit#pairs by AST node; CLEAR carries it as a list of pairs + # (a map keyed by the recursive Locatable union closes a type cycle Zig + # cannot size). The WIRE format stays Ruby's Hash encoding. + if (tup = bare[/\A\[\]Tuple<(.+?),\s*(.+)>\z/, 0]) + kt = bare[/\A\[\]Tuple<(.+?),\s*(.+)>\z/, 1] + vt = bare[/\A\[\]Tuple<(.+?),\s*(.+)>\z/, 2] + kpre, kexp = clear_value_encoder(kt, "#{slot}_k", fields, "#{slot}k") + vpre, vexp = clear_value_encoder(vt, "#{slot}_v", fields, "#{slot}v") + kbody = kpre.empty? ? "" : kpre + "\n" + vbody = vpre.empty? ? "" : vpre + "\n" + return [ + " MUTABLE #{slot}_pairs: String[] = [];\n" \ + " MUTABLE #{slot}_n = 0;\n" \ + " WHILE #{slot}_n < #{expr}.length() DO\n" \ + " #{slot}_k, #{slot}_v = UNWRAP (#{expr}[#{slot}_n]);\n#{kbody}#{vbody}" \ + " &#{slot}_pairs.append(#{kexp} $+ #{vexp});\n" \ + " #{slot}_n += 1;\n" \ + " END\n" \ + " #{slot}_pairs = #{slot}_pairs |> ORDER_BY _;\n" \ + " MUTABLE #{slot} = \"H\" $+ #{slot}_pairs.length().toString() $+ \"[\" $+ #{slot}_pairs.join(\"\") $+ \"]\";", + slot + ] + end + + if (elem = bare[/\A\[\](.+)\z/, 1]) + pre, inner = clear_value_encoder(elem, "#{slot}_item", fields, "#{slot}i") + body = pre.empty? ? "" : pre + "\n" + return [ + " MUTABLE #{slot} = \"A\" $+ #{expr}.length().toString() $+ \"[\";\n" \ + " MUTABLE #{slot}_n = 0;\n" \ + " WHILE #{slot}_n < #{expr}.length() DO\n" \ + " #{slot}_item = #{expr}[#{slot}_n]?;\n#{body}" \ + " #{slot} = #{slot} $+ #{inner};\n" \ + " #{slot}_n += 1;\n" \ + " END\n" \ + " #{slot} = #{slot} $+ \"]\";", + slot + ] + end + + # A `[Set]T` value has no canonical_encode case on the Ruby side, so there + # is no wire format to match. Panic rather than invent one; the smoke + # corpus leaves these slots empty, and a mismatch should be loud. + if bare.start_with?('[Set]') + return [" panic(\"parser compat: no wire format for #{bare}\");", '""'] + end + + if (m = bare.match(/\A\{(.+?)\}(.+)\z/)) + kpre, kexp = clear_value_encoder(m[1], "#{slot}_k", fields, "#{slot}k") + vpre, vexp = clear_value_encoder(m[2], "#{slot}_v", fields, "#{slot}v") + kbody = kpre.empty? ? "" : kpre + "\n" + vbody = vpre.empty? ? "" : vpre + "\n" + return [ + " MUTABLE #{slot}_pairs: String[] = [];\n" \ + " #{expr}.keys() |> EACH {\n" \ + " #{slot}_k = _;\n" \ + " #{slot}_v = #{expr}[_]?;\n#{kbody}#{vbody}" \ + " &#{slot}_pairs.append(#{kexp} $+ #{vexp});\n" \ + " };\n" \ + " #{slot}_pairs = #{slot}_pairs |> ORDER_BY _;\n" \ + " MUTABLE #{slot} = \"H\" $+ #{slot}_pairs.length().toString() $+ \"[\" $+ #{slot}_pairs.join(\"\") $+ \"]\";", + slot + ] + end + + simple = + if bare == 'TokenValue' then "encodeTokenValue(#{expr})" + elsif bare == 'Token' then "encodeToken(#{expr})" + elsif bare == 'Type' then "encodeType(#{expr})" + elsif bare == 'Locatable' then "encodeLocatable(#{expr})" + elsif bare == 'ContractClauseValue' then "encodeContractClauseValue(#{expr})" + elsif bare == 'PassStateValue' then "encodePassStateValue(#{expr})" + elsif @generated_root && clear_union_variants(@generated_root).key?(bare) + (@union_encoders_needed ||= Set.new) << bare + "encode#{bare}(#{expr})" + elsif type.to_s.include?('@symbol') then "lengthEncoded(\"Y\", CAST(#{expr} AS String))" + elsif bare == 'String' then "lengthEncoded(\"S\", #{expr})" + elsif %w[Int64 UInt64].include?(bare) then "(\"I\" $+ #{expr}.toString() $+ \";\")" + elsif bare == 'Float64' then "(\"F\" $+ floatValueText(#{expr}) $+ \";\")" + elsif bare == 'Bool' then "(IF #{expr} THEN \"B1\" ELSE \"B0\" END)" + elsif fields.key?(bare) then "encode#{bare}(#{expr})" + end + + simple ? ['', simple] : nil + end + + def struct_class_for(name) + [AST, Object].each do |scope| + next unless scope.const_defined?(name, false) + candidate = scope.const_get(name, false) + next unless candidate.is_a?(Class) + return candidate if candidate < Struct || candidate.respond_to?(:props) + end + nil + end + + # An emitted encoder can reference a struct the corpus never instantiated + # (an always-empty Capture list, say). Close over those so every referenced + # type has an encoder. + def close_over_referenced_types!(classes, fields) + loop do + added = false + classes.keys.each do |name| + (fields[name] || {}).each_value do |decl| + base = clear_base_type(decl).sub(/\A\?/, '').sub(/\A\[\]/, '').sub(/\A\{[^}]*\}/, '') + next if classes.key?(base) || !fields.key?(base) + klass = struct_class_for(base) + next unless klass + classes[base] = klass + (@closure_added ||= Set.new) << base + added = true + end + end + break unless added + end + end + + def node_encoders(cases, generated_root) + @generated_root = generated_root + fields = clear_struct_fields(generated_root) + classes = corpus_node_classes(cases) + close_over_referenced_types!(classes, fields) + emitted = [] + unsupported = [] + + classes.sort.each do |name, klass| + decls = fields[name] + next unless decls + + # Reached only through a collection the corpus never populates, so this + # encoder is dead. Emit it so the referencing encoder compiles, and panic + # rather than invent an encoding that was never exercised. + if (@closure_added ||= Set.new).include?(name) + emitted << "PRIVATE FN encode#{name}(node: #{name}) RETURNS String EFFECTS REENTRANT ->\n" \ + " panic(\"parser compat: #{name} reached but never encoded\");\n" \ + " RETURN \"\";\nEND" + next + end + + members = struct_member_names(klass).reject { |m| STAMP_FIELDS.include?(m) }.sort + parts = members.each_with_index.map do |member, slot_index| + decl = decls[member] + pair = decl && clear_value_encoder(decl, "node.#{member}", fields, "f#{slot_index}") + if pair.nil? && @never_populated[name][member] + # The translation left this field untyped (Any) and the corpus never + # populates it. Encode the nil Ruby also emits, but assert it rather + # than assume -- a populated field must fail loudly, not silently + # diverge. + # `== NIL` on an `Any@multiowned` slot compares the PAYLOAD (Any + # resolves to f64) rather than the optional, so ask with EXISTS. + # A NON-optional untyped slot cannot be asked at all -- it always + # holds something -- so encode the nil Ruby emits and say so. + pair = if clear_base_type(decl).to_s.start_with?('?') + [" IF node.#{member} EXISTS AS untyped_#{member}_set THEN\n" \ + " ASSERT FALSE, \"parser compat: #{name}.#{member} is populated but untyped\";\n" \ + " END", '"N"'] + else + [" # #{name}.#{member} is untyped and non-optional: unaskable here.", '"N"'] + end + end + unless pair + unsupported << "#{name}.#{member} (#{decl.inspect})" + next nil + end + prelude, expr = pair + line = " out = out $+ lengthEncoded(\"S\", #{member.inspect}) $+ #{expr};" + prelude.empty? ? line : "#{prelude}\n#{line}" + end + next if parts.any?(&:nil?) + + emitted << <<~FN.chomp + PRIVATE FN encode#{name}(node: #{name}) RETURNS String EFFECTS REENTRANT -> + MUTABLE out = "O#{name.bytesize}:#{name}#{members.length}["; + #{parts.join("\n")} + RETURN out $+ "]"; + END + FN + end + + raise "parser compat: cannot encode #{unsupported.join(', ')}" if unsupported.any? + + [emitted.join("\n\n"), [locatable_dispatch(classes.keys, fields, generated_root), + union_encoders(generated_root, fields, classes.keys.to_set)].reject(&:empty?).join("\n\n")] + end + + # One encoder per declared union: dispatch on the active variant and encode + # its payload. Locatable keeps its hand-written dispatch (it names every AST + # node and only the corpus-reachable ones get encoders). + def union_encoders(generated_root, fields, encodable) + clear_union_variants(generated_root).filter_map do |name, variants| + next if name == 'Locatable' + next unless @union_encoders_needed&.include?(name) + # A union that already carries a Locatable variant does not need a + # per-node arm as well: encodeLocatable dispatches those. Keeping them + # expands one encoder into ~130 arms and pulls in every node encoder. + covers_nodes = variants.any? { |_, payload| payload == 'Locatable' } + node_variants = covers_nodes ? locatable_variants(generated_root) : Set.new + arms = variants.filter_map do |variant, payload| + next if covers_nodes && node_variants.include?(payload) + # A variant the corpus never produces has no encoder to call. Leaving + # its arm out drops it into the panic below, which is the same contract + # the never-populated struct encoders use. + bare = payload.sub(/\A\?/, '').sub(/\A\[\]/, '').sub(/\A\{[^}]*\}/, '').sub(/@\w+\z/, '') + next if fields.key?(bare) && !encodable.include?(bare) + slot = "u_#{name.downcase}_#{variant.downcase}" + pre, expr = clear_value_encoder(payload, slot, fields, "u#{name}#{variant}") + next if expr.nil? + body = pre.to_s.empty? ? "" : "#{pre}\n" + " IF node IS_A #{name}.#{variant} AS #{slot} THEN\n#{body} RETURN #{expr};\n END" + end + "PRIVATE FN encode#{name}(node: #{name}) RETURNS String EFFECTS REENTRANT ->\n" \ + "#{arms.join("\n")}\n" \ + " panic(\"parser compat: unsupported #{name} variant\");\nEND" + end.join("\n\n") + end + + def locatable_variants(generated_root) + src = File.read(File.join(generated_root, 'ast', 'ast.clear')) + src[/^(?:PUB )?UNION Locatable \{(.*?)\}/m, 1].to_s.scan(/(\w+):/).flatten.to_set + end + + def locatable_dispatch(names, fields, generated_root) + variants = locatable_variants(generated_root) + arms = names.select { |n| fields.key?(n) && variants.include?(n) }.sort.map do |n| + " IF node IS_A Locatable.#{n} AS item THEN RETURN encode#{n}(item); END" + end + <<~FN.chomp + PRIVATE FN encodeLocatable(node: Locatable) RETURNS String EFFECTS REENTRANT -> + #{arms.join("\n")} + panic("parser compat: unsupported AST node"); + END + + PRIVATE FN encodePassStateValue(node: PassStateValue) RETURNS String EFFECTS REENTRANT -> + panic("parser compat: pass state reached but never populated by the parser"); + END + + PRIVATE FN encodeContractClauseValue(node: ContractClauseValue) RETURNS String EFFECTS REENTRANT -> + IF node IS_A String AS text THEN RETURN lengthEncoded("S", text); END + IF node IS_A Locatable AS item THEN RETURN encodeLocatable(item); END + panic("parser compat: unsupported contract clause value"); + END + FN + end + def clear_harness_source(cases, generated_root) parser_path = parser_require_spec(generated_root) + type_path = type_require_spec(generated_root) + lexer_path = lexer_require_spec(generated_root) + encoders, dispatch = node_encoders(cases, generated_root) + node_encoders_source = "#{encoders}\n\n#{dispatch}\n" + calls = cases.each_with_index.map do |entry, index| - " dumpCase(#{LexerHarnessSupport.clear_string_expr(entry['source'])}, #{index}, #{LexerHarnessSupport.clear_string_expr(entry['name'])}) OR_ELSE RAISE;" + " dumpCase(#{LexerHarnessSupport.clear_string_expr(entry['source'])}, #{index}, #{LexerHarnessSupport.clear_string_expr(entry['name'])}) OR_ELSE reportCaseFailure(#{index}, #{LexerHarnessSupport.clear_string_expr(entry['name'])});" end.join("\n") <<~CLEAR REQUIRE #{LexerHarnessSupport.clear_string_literal(parser_path)}; + REQUIRE #{LexerHarnessSupport.clear_string_literal(type_path)}; + REQUIRE #{LexerHarnessSupport.clear_string_literal(lexer_path)}; PRIVATE FN escapeCompat(value: String) RETURNS String -> MUTABLE out = ""; MUTABLE i = 0; WHILE i < value.length() DO - ch = value.charAt(i); + MUTABLE ch = value.charAt(i); IF ch == "\\\\" THEN out = out $+ "\\\\\\\\"; ELSE_IF ch == "\\n" THEN @@ -460,14 +894,19 @@ def clear_harness_source(cases, generated_root) RETURN prefix $+ whole.toString() $+ "." $+ trimTrailingZeros(frac_text); END - PRIVATE FN encodeTokenValue(value: TokenValue) RETURNS String -> - RETURN MATCH value START - TokenValue.Nil -> "N", - TokenValue.Str AS item -> lengthEncoded("S", item), - TokenValue.Int AS item -> "I" $+ item.toString() $+ ";", - TokenValue.UInt AS item -> "I" $+ item.toString() $+ ";", - TokenValue.Float AS item -> "F" $+ floatValueText(item) $+ ";", - END; + PRIVATE FN encodeTokenValue(value: ?TokenValue) RETURNS String -> + IF value EXISTS AS payload THEN + IF payload IS_A TokenValue.StringValue AS item THEN RETURN lengthEncoded("S", item); END + IF payload IS_A TokenValue.Int64Value AS item THEN RETURN "I" $+ item.toString() $+ ";"; END + IF payload IS_A TokenValue.Float64Value AS item THEN RETURN "F" $+ floatValueText(item) $+ ";"; END + IF payload IS_A TokenValue.BoolValue AS item THEN RETURN IF item THEN "B1" ELSE "B0" END; END + END + RETURN "N"; + END + + PRIVATE FN encodeType(value: Type) RETURNS String -> + RETURN "O4:Type1[" $+ lengthEncoded("S", "resolved") $+ + lengthEncoded("Y", CAST(type__resolved(value) AS String)) $+ "]"; END PRIVATE FN encodeToken(token: Token) RETURNS String -> @@ -479,62 +918,17 @@ def clear_harness_source(cases, generated_root) "]"; END - PRIVATE FN encodeCompat(value: Any) RETURNS String EFFECTS REENTRANT -> - IF value == NIL THEN - RETURN "N"; - ELSE_IF value IS_A Token AS token THEN - RETURN encodeToken(token); - ELSE_IF value IS_A String@symbol AS symbol_value THEN - RETURN lengthEncoded("Y", CAST(symbol_value AS String)); - ELSE_IF value IS_A String AS string_value THEN - RETURN lengthEncoded("S", string_value); - ELSE_IF value IS_A Bool AS bool_value THEN - RETURN IF bool_value THEN "B1" ELSE "B0" END; - ELSE_IF value IS_A Int64 AS int_value THEN - RETURN "I" $+ int_value.toString() $+ ";"; - ELSE_IF value IS_A UInt64 AS uint_value THEN - RETURN "I" $+ uint_value.toString() $+ ";"; - ELSE_IF value IS_A Float64 AS float_value THEN - RETURN "F" $+ floatValueText(float_value) $+ ";"; - ELSE_IF value IS_A Any[] AS items THEN - MUTABLE encoded = "A" $+ items.length().toString() $+ "["; - MUTABLE i = 0; - WHILE i < items.length() DO - encoded = encoded $+ encodeCompat(items[i]); - i += 1; - END - RETURN encoded $+ "]"; - ELSE_IF value IS_A HashMap AS values THEN - MUTABLE pairs: String[] = []; - values.keys() |> EACH { - pairs.append(encodeCompat(_) $+ encodeCompat(values[_])); - }; - pairs = pairs.sort(); - RETURN "H" $+ pairs.length().toString() $+ "[" $+ pairs.join("") $+ "]"; - ELSE_IF value IS_A Struct AS object THEN - MUTABLE members = object.class().members().sort(); - MUTABLE encoded = "O" $+ object.class().name().length().toString() $+ ":" $+ - object.class().name() $+ members.length().toString() $+ "["; - MUTABLE i = 0; - WHILE i < members.length() DO - member = members[i]; - encoded = encoded $+ lengthEncoded("S", member) $+ encodeCompat(object[member]); - i += 1; - END - RETURN encoded $+ "]"; - END - panic("unsupported parser compatibility value"); +#{node_encoders_source} + # One failing case used to abort the whole run, hiding every case after it. + PRIVATE FN reportCaseFailure(index: Int64, name: String) RETURNS Void -> + print("CASE|" $+ index.toString() $+ "|" $+ escapeCompat(name) $+ "|error|parse_failed"); + print("ENDCASE"); + RETURN; END - PRIVATE FN dumpCase(source: String@raw, index: Int64, name: String) RETURNS !Void -> - tokens = tokenizeSource(source) OR_ELSE RAISE; - MUTABLE parser = clearParser__new(tokens, source); - program = parse(parser); - IF program == NIL THEN - panic("parser returned NIL"); - END + program = clearParser__parse_source(CAST(source AS String)) OR_ELSE RAISE; print("CASE|" $+ index.toString() $+ "|" $+ escapeCompat(name) $+ "|ok|"); - print("AST|" $+ escapeCompat(encodeCompat(program?))); + print("AST|" $+ escapeCompat(encodeProgram(program))); print("ENDCASE"); RETURN; END @@ -574,6 +968,12 @@ def parse_clear_output(stdout) current.delete('index') cases << current current = nil + when /\A\[Scheduler\]/, /\A(Segmentation fault|thread \d+ panic|Aborted)/ + # The CLEAR side aborted partway -- a scheduler error, or a crash that + # takes the process down. Report what it DID produce so the cases that + # work can still be byte-compared. + warn "parser_compat: CLEAR aborted after #{cases.length} case(s): #{line}" + break else raise "unexpected CLEAR parser output: #{line}" end diff --git a/tools/selfhost_build.sh b/tools/selfhost_build.sh index 6ca14116b..c14b84ec1 100755 --- a/tools/selfhost_build.sh +++ b/tools/selfhost_build.sh @@ -5,9 +5,21 @@ set -uo pipefail cd /home/yahn/cheat -if [ -e compiler/.ruby-original ]; then - echo "compiler/.ruby-original exists -- a previous run died mid-swap." >&2 - echo "Inspect it, then: rm -rf compiler/ruby && mv compiler/.ruby-original compiler/ruby" >&2 +# An interrupted run leaves the Sorbet-stripped mirror sitting where +# compiler/ruby belongs. The state is recognizable -- the mirror has no sigs -- +# and healing it is exactly what the EXIT trap would have done, so do that +# rather than block every later build. +if [ -f compiler/ruby/ast/type.rb ] && ! grep -q '^ sig {' compiler/ruby/ast/type.rb; then + echo "[selfhost] restoring compiler/ruby after an interrupted run" >&2 + if [ -e compiler/.ruby-original ]; then + rm -rf compiler/ruby && mv compiler/.ruby-original compiler/ruby + else + # The saved copy is gone too; compiler/ruby is fully tracked, so git has it. + git checkout -- compiler/ruby || exit 1 + fi +elif [ -e compiler/.ruby-original ]; then + echo "compiler/.ruby-original exists and compiler/ruby is NOT the mirror." >&2 + echo "Inspect both, then keep the one you want as compiler/ruby." >&2 exit 1 fi diff --git a/transpile-tests/186_string_replace_case.clear b/transpile-tests/186_string_replace_case.clear index 8e2fdbf59..d46fad32b 100644 --- a/transpile-tests/186_string_replace_case.clear +++ b/transpile-tests/186_string_replace_case.clear @@ -1,4 +1,4 @@ -# Test: replace, downcase, upcase string functions. +# Test: replace, downcase, upcase, capitalize string functions. FN main() RETURNS Void -> # replace: all occurrences @@ -19,9 +19,17 @@ FN main() RETURNS Void -> ASSERT "ALREADY".upcase() == "ALREADY"; ASSERT "MiXeD123".upcase() == "MIXED123"; + # capitalize + ASSERT "hello world".capitalize() == "Hello world"; + ASSERT "hELLO WORLD".capitalize() == "Hello world"; + ASSERT "Already".capitalize() == "Already"; + ASSERT "".capitalize() == ""; + ASSERT "9lives".capitalize() == "9lives"; + # chained result = replace("Hello WORLD".downcase(), "hello", "hi"); ASSERT result == "hi world"; + ASSERT "warning".capitalize().upcase() == "WARNING"; print("PASS"); RETURN; diff --git a/transpile-tests/23_optional.clear b/transpile-tests/23_optional.clear index d7baee5af..02a3a72a3 100644 --- a/transpile-tests/23_optional.clear +++ b/transpile-tests/23_optional.clear @@ -13,5 +13,25 @@ FN main() RETURNS Void -> END ASSERT result == 1, "Conditional on optional should work"; + + # Two optionals compare like Rust's Option, Swift's Optional and Kotlin's + # nullable: both absent is equal, one absent is not, both present compares + # the payloads. + other_num: ?Int64 = 42; + also_empty: ?Int64 = NIL; + ASSERT maybe_num == other_num, "present optionals with equal payloads are equal"; + ASSERT maybe_num != empty, "a present optional never equals an absent one"; + ASSERT empty == also_empty, "two absent optionals are equal"; + ASSERT !(empty != also_empty), "two absent optionals are not unequal"; + + # Strings take the content-comparison path rather than Zig's ==. + word: ?String = "clear"; + same_word: ?String = "clear"; + other_word: ?String = "zig"; + no_word: ?String = NIL; + ASSERT word == same_word, "present string optionals compare by content"; + ASSERT word != other_word, "different string payloads are unequal"; + ASSERT word != no_word, "a present string never equals an absent one"; + ASSERT no_word == NIL, "an absent optional still compares against NIL"; RETURN; END diff --git a/transpile-tests/667_map_keys_values_return_type.clear b/transpile-tests/667_map_keys_values_return_type.clear index 77f9976c7..85ad3a2b2 100644 --- a/transpile-tests/667_map_keys_values_return_type.clear +++ b/transpile-tests/667_map_keys_values_return_type.clear @@ -5,7 +5,10 @@ # comparison then rejected `RETURN k;` with a RETURN_MISMATCH whose two # sides printed identically. -FN codes(diagnostics: {String@symbol}Int64) RETURNS []String@symbol -> +# keys() returns the map's OWN keys -- duplicated on insert, freed at deinit -- +# so they are owned Strings even though lookups use interned symbol handles. +# `String@symbol` is a distinct type now, and it would be a lie here. +FN codes(diagnostics: {String@symbol}Int64) RETURNS []String -> k = diagnostics.keys(); RETURN k; END diff --git a/transpile-tests/900_borrow_through_with_alias.clear b/transpile-tests/900_borrow_through_with_alias.clear new file mode 100644 index 000000000..685a8a123 --- /dev/null +++ b/transpile-tests/900_borrow_through_with_alias.clear @@ -0,0 +1,48 @@ +# RETURNS self:T lets an accessor hand back a borrow scoped to its receiver. +# Reading through a WITH POLYMORPHIC alias must be accepted: the alias is +# already a scoped borrow of its source. This failed with +# MUTABLE_PARAM_NEEDS_RESTRICT on any call through the alias. + +STRUCT Cursor { items: []Int64, pos: Int64 } + +PUB FN cursor__at(self: Cursor) RETURNS self: Int64 + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS view { + RETURN UNWRAP (view.items[view.pos]); +} +END + +PUB FN is_zero(v: Int64) RETURNS Bool -> + RETURN (v == 0_i64); +END + +PUB FN cursor__has_more(self: Cursor) RETURNS Bool + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS view { + RETURN (view.pos < 3_i64); +} +END + +PUB FN cursor__advance(MUTABLE self: Cursor) RETURNS Int64 + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + # first statement of the block, borrow nested in an argument inside AND + IF (cursor__has_more(view) AND is_zero(cursor__at(view))) THEN + view.pos = (view.pos + 1_i64); + RETURN 0_i64; + END + MUTABLE held = COPY cursor__at(view); + view.pos = (view.pos + 1_i64); + RETURN held; +} +END + +FN main() RETURNS Void -> + MUTABLE c = Cursor{ items: [0_i64, 5_i64, 0_i64], pos: 0_i64 }; + ASSERT cursor__advance(&c) == 0_i64, "first is zero"; + ASSERT cursor__advance(&c) == 5_i64, "second is five"; + print("ok"); +END diff --git a/transpile-tests/901_block_tail_hoists.clear b/transpile-tests/901_block_tail_hoists.clear new file mode 100644 index 000000000..b32540706 --- /dev/null +++ b/transpile-tests/901_block_tail_hoists.clear @@ -0,0 +1,15 @@ +# A block expression's tail materializations must stay INSIDE the block: they +# can reference locals the block declares. lower_block_expr left them in +# function_state.pending_stmts, so the enclosing statement flushed them ABOVE +# the block and the generated Zig used `x` before its declaration. + +FN pair(a: String, b: String) RETURNS Tuple -> + RETURN ( { MUTABLE x = COPY a; MUTABLE y = COPY b; Tuple{COPY x, COPY y} } ); +END + +FN main() RETURNS Void -> + first, second = pair("hi", "there"); + ASSERT first == "hi", "first"; + ASSERT second == "there", "second"; + print("ok"); +END diff --git a/transpile-tests/902_alloc_call_in_control_condition.clear b/transpile-tests/902_alloc_call_in_control_condition.clear new file mode 100644 index 000000000..6b326b5b7 --- /dev/null +++ b/transpile-tests/902_alloc_call_in_control_condition.clear @@ -0,0 +1,42 @@ +# An allocating call nested inside a control condition (`f(x).field == y`) must +# be hoisted. The normalizer walked only Struct-based MIR nodes, so anything +# under a T::Struct node -- RegistryCall, which is what `==` on symbols lowers +# to -- was invisible and the call reached the checker unhoisted +# (UNHOISTED_ALLOC). The WHILE form also proves the hoist stays per-iteration. + +STRUCT Tok { text: String, kind: String@symbol } +STRUCT Stream { toks: []Tok, pos: Int64 } + +PUB FN stream__current(self: Stream) RETURNS Tok + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS view { + RETURN COPY UNWRAP (view.toks[view.pos]); +} +END + +PUB FN stream__count_words(MUTABLE self: Stream) RETURNS Int64 + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + MUTABLE n = 0_i64; + WHILE (stream__current(view).kind != :eof) DO + IF (stream__current(view).kind == :word) THEN + n = (n + 1_i64); + END + view.pos = (view.pos + 1_i64); + END + RETURN n; +} +END + +FN main() RETURNS Void -> + MUTABLE s = Stream{ toks: [ + Tok{ text: "a", kind: :word }, + Tok{ text: "+", kind: :op }, + Tok{ text: "b", kind: :word }, + Tok{ text: "", kind: :eof } + ], pos: 0_i64 }; + ASSERT stream__count_words(&s) == 2_i64, "two words before eof"; + print("ok"); +END diff --git a/transpile-tests/903_borrow_return_heap_carry.clear b/transpile-tests/903_borrow_return_heap_carry.clear new file mode 100644 index 000000000..7446e5164 --- /dev/null +++ b/transpile-tests/903_borrow_return_heap_carry.clear @@ -0,0 +1,30 @@ +# A `RETURNS self:T` accessor whose element type carries heap storage (String) +# still returns a BORROW: the caller's argument owns it. Escape analysis used to +# mark the return heap-carried anyway, so the call was treated as owned and +# every use of it outside a Let init failed with UNHOISTED_ALLOC. + +STRUCT Tok { text: String, kind: String@symbol } +STRUCT Stream { toks: []Tok, pos: Int64 } + +PUB FN stream__current(self: Stream) RETURNS self: Tok + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS view { + RETURN UNWRAP (view.toks[view.pos]); +} +END + +PUB FN stream__at_end?(self: Stream) RETURNS Bool + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS view { + RETURN (stream__current(view).kind == :eof); +} +END + +FN main() RETURNS Void -> + s = Stream{ toks: [Tok{ text: "a", kind: :word }, Tok{ text: "", kind: :eof }], pos: 1_i64 }; + ASSERT stream__at_end?(s), "second token ends the stream"; + ASSERT stream__current(s).text == "", "borrowed token reads through"; + print("ok"); +END diff --git a/transpile-tests/904_frame_element_orelse_cleanup.clear b/transpile-tests/904_frame_element_orelse_cleanup.clear new file mode 100644 index 000000000..2bbcea4cd --- /dev/null +++ b/transpile-tests/904_frame_element_orelse_cleanup.clear @@ -0,0 +1,15 @@ +# A binding initialized from a frame-placed element view (`list[i] OR_ELSE ""`) +# inherits a cleanup recipe from the init expression. That recipe named the +# heap allocator while the AllocMark named :frame, so the binding had two +# allocators (ALLOC_CLEANUP_MISMATCH). One binding, one allocator. + +FN line_at(source: String, index: Int64) RETURNS Int64 -> + line_text = (source.split("\n")[index] OR_ELSE ""); + RETURN line_text.length(); +END + +FN main() RETURNS Void -> + ASSERT line_at("ab\ncde", 1) == 3_i64, "second line"; + ASSERT line_at("ab\ncde", 7) == 0_i64, "missing line"; + print("ok"); +END diff --git a/transpile-tests/905_mutable_arg_writeback_ownership.clear b/transpile-tests/905_mutable_arg_writeback_ownership.clear new file mode 100644 index 000000000..b4857daf3 --- /dev/null +++ b/transpile-tests/905_mutable_arg_writeback_ownership.clear @@ -0,0 +1,29 @@ +# COPY through a borrow must deep-copy what the pointee owns, and a MUTABLE +# param whose argument is a temporary must write back into the binding that +# owns it. Both were wrong here: `COPY item` on a MATCH payload bit-copied +# (aliasing the String), and the temp was copied into a separate mutable slot +# whose value nobody freed -- one leak plus a double free. + +STRUCT Alpha { file: ?String, name: String } +STRUCT Beta { count: Int64 } +UNION Item { Alpha: Alpha, Beta: Beta } + +FN stamp(MUTABLE node: Item, src: ?String) RETURNS Item -> + MATCH node START + Item.Alpha AS item -> + MUTABLE item_mutable = COPY item; + item_mutable.file = COPY src; + node = Item{ Alpha: item_mutable };, + Item.Beta AS item -> + PASS + END + RETURN node; +END + +FN main() RETURNS Void -> + out = stamp(Item{ Alpha: Alpha{ file: NIL, name: "a" } }, "x"); + IF out IS_A Alpha AS a THEN + ASSERT UNWRAP (a.file) == "x", "stamped"; + END + print("ok"); +END diff --git a/transpile-tests/906_block_result_guarded_transfer.clear b/transpile-tests/906_block_result_guarded_transfer.clear new file mode 100644 index 000000000..9606d123c --- /dev/null +++ b/transpile-tests/906_block_result_guarded_transfer.clear @@ -0,0 +1,32 @@ +# A block expression that hands out a guarded-cleanup binding must set the +# `_moved` flag as it breaks: the TransferMark carried no MoveMark, so the +# value was transferred out AND cleaned up on the way (OWNERSHIP_IMPLICIT_MOVE). + +STRUCT Tok { file: ?String, line: Int64 } +STRUCT Span { file: ?String, line: Int64 } +STRUCT Alpha { source_range: ?Span, name: String } +STRUCT Beta { source_range: ?Span, count: Int64 } +UNION Item { Alpha: Alpha, Beta: Beta } + +FN stamp(MUTABLE node: Item, first: Tok, last: Tok) RETURNS Item -> + MATCH node START + Item.Alpha AS item -> + MUTABLE item_mutable = COPY item; + item_mutable.source_range = Span{ file: ( { MUTABLE src: ?String = first.file; MUTABLE res: ?String = NIL; IF src != NIL THEN res = COPY src; ELSE res = last.file; END res } ), line: first.line }; + node = Item{ Alpha: item_mutable };, + Item.Beta AS item -> + MUTABLE item_mutable = COPY item; + item_mutable.source_range = Span{ file: ( { MUTABLE src: ?String = first.file; MUTABLE res: ?String = NIL; IF src != NIL THEN res = COPY src; ELSE res = last.file; END res } ), line: first.line }; + node = Item{ Beta: item_mutable }; + END + RETURN node; +END + +FN main() RETURNS Void -> + out = stamp(Item{ Alpha: Alpha{ source_range: NIL, name: "a" } }, Tok{ file: "x", line: 1_i64 }, Tok{ file: "y", line: 2_i64 }); + IF out IS_A Alpha AS a THEN + sr = UNWRAP (a.source_range); + ASSERT UNWRAP (sr.file) == "x", "stamped"; + END + print("ok"); +END diff --git a/transpile-tests/907_pipeline_result_returned.clear b/transpile-tests/907_pipeline_result_returned.clear new file mode 100644 index 000000000..0d7616874 --- /dev/null +++ b/transpile-tests/907_pipeline_result_returned.clear @@ -0,0 +1,25 @@ +# A pipeline result bound to a local that is then RETURNED: escape analysis +# promotes the binding to the heap, and the pipeline's accumulator must be +# promoted with it. The rewriter only gave the accumulator a SymbolEntry when +# it could already see a heap destination, so there was nothing for escape +# analysis to promote and the accumulator stayed frame-allocated +# (OWNED_RESULT_ALLOC_MISMATCH). + +FN strip(value: String) RETURNS String -> + RETURN value.substr(1_i64, (value.length() - 1_i64)); +END + +FN table() RETURNS {String}Int64 -> + RETURN {"@a": 1_i64, "@b": 2_i64}; +END + +FN names() RETURNS []String -> + candidates = table().keys() |> SELECT strip(_); + RETURN candidates; +END + +FN main() RETURNS Void -> + n = names(); + ASSERT n.length() == 2_i64, "two"; + print("ok"); +END diff --git a/transpile-tests/908_lambda_tail_owned_block.clear b/transpile-tests/908_lambda_tail_owned_block.clear new file mode 100644 index 000000000..81c39853c --- /dev/null +++ b/transpile-tests/908_lambda_tail_owned_block.clear @@ -0,0 +1,29 @@ +# A lambda's tail expression is its return value, but the RETURN is synthesized +# during MIR lowering. The return-value hoist ran before ownership +# finalization, and finalization is what makes a value block read as owned -- +# so the hoist skipped it and the block reached the checker unhoisted +# (UNHOISTED_ALLOC). The same expression in a plain FN was always fine. + +STRUCT Cap { name: String, tag: String } + +FN fallback() RETURNS !String -> + RETURN "anon"; +END + +FN apply(p: ?String, blk: FN(?String) -> Elem) RETURNS !Elem -> + RETURN blk(p); +END + +FN via_lambda(p: ?String) RETURNS !Cap -> + RETURN TRY (apply(p, %(v: ?String) -> { + Cap{ name: "n", tag: COPY (v OR_ELSE TRY (fallback())) } + })); +END + +FN main() RETURNS !Void -> + a = TRY (via_lambda(NIL)); + ASSERT a.tag == "anon", "fallback"; + b = TRY (via_lambda("x")); + ASSERT b.tag == "x", "present"; + print("ok"); +END diff --git a/transpile-tests/909_fn_type_mutable_param.clear b/transpile-tests/909_fn_type_mutable_param.clear new file mode 100644 index 000000000..dad9e6e4a --- /dev/null +++ b/transpile-tests/909_fn_type_mutable_param.clear @@ -0,0 +1,33 @@ +# A callback that mutates what it is handed: `FN(MUTABLE T) -> R`. Function +# types could not express a mutable parameter at all, so a block could only +# read -- the call site's `&` was rejected as 'not MUTABLE'. + +STRUCT Counter { n: Int64 } + +FN bump_twice(MUTABLE self: Counter, blk: FN(MUTABLE Counter) -> Elem) RETURNS []Elem + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + MUTABLE out: []Elem = List[]; + &out.append(blk(&view)); + &out.append(blk(&view)); + RETURN out; +} +END + +FN step(MUTABLE self: Counter) RETURNS Int64 + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + view.n = (view.n + 1_i64); + RETURN view.n; +} +END + +FN main() RETURNS Void -> + MUTABLE c = Counter{ n: 0_i64 }; + got = bump_twice(&c, %(MUTABLE view: Counter) -> step(&view)); + ASSERT got.length() == 2_i64, "two"; + ASSERT UNWRAP (got[1_i64]) == 2_i64, "second is 2"; + print("ok"); +END diff --git a/transpile-tests/910_lambda_tail_placement.clear b/transpile-tests/910_lambda_tail_placement.clear new file mode 100644 index 000000000..5d28ff2fe --- /dev/null +++ b/transpile-tests/910_lambda_tail_placement.clear @@ -0,0 +1,28 @@ +# A lambda's tail value leaves the lambda's frame, so it must be built on the +# heap. The synthesized RETURN is created during MIR lowering, after escape +# analysis, so nothing had made that placement decision: the hoisted return +# binding was heap while the value itself was built on the frame +# (OWNED_RESULT_ALLOC_MISMATCH). + +FN collect(count: Int64, blk: FN(Int64) -> Elem) RETURNS ![]Elem -> + MUTABLE items: []Elem = List[]; + MUTABLE i = 0_i64; + WHILE (i < count) DO + &items.append(blk(i)); + i = (i + 1_i64); + END + RETURN items; +END + +FN labels(n: Int64) RETURNS ![]Tuple -> + RETURN TRY (collect(n, %(i: Int64) -> { + word = "item"; + Tuple{COPY word, COPY word} + })); +END + +FN main() RETURNS !Void -> + out = TRY (labels(2_i64)); + ASSERT out.length() == 2_i64, "two"; + print("ok"); +END diff --git a/transpile-tests/911_map_literal_value_placement.clear b/transpile-tests/911_map_literal_value_placement.clear new file mode 100644 index 000000000..a6d8dc64c --- /dev/null +++ b/transpile-tests/911_map_literal_value_placement.clear @@ -0,0 +1,20 @@ +# A map literal owns its values, so a value has to live in the map's own +# allocator. The list literal places its elements that way; the map literal +# stored the @rodata pointer of a string literal directly, so the map's +# cleanup freed read-only memory ("Invalid free" at runtime). + +FN note(fields: {String@symbol}String) RETURNS Int64 -> + RETURN fields.length(); +END + +FN main() RETURNS Void -> + item = "a"; + # Map literal bound to a local: values must live in the map's allocator. + m: {String@symbol}String = {:name: "a"}; + ASSERT note(m) == 1_i64, "one entry"; + + # ... and the same literal passed straight as an argument. + ASSERT note({:name: "a", :kind: "dup"}) == 2_i64, "two entries"; + ASSERT note({:name: COPY item, :kind: "dup"}) == 2_i64, "mixed owned and literal"; + print("ok"); +END diff --git a/transpile-tests/912_each_body_frame_rewind.clear b/transpile-tests/912_each_body_frame_rewind.clear new file mode 100644 index 000000000..60fb0e7db --- /dev/null +++ b/transpile-tests/912_each_body_frame_rewind.clear @@ -0,0 +1,25 @@ +# An EACH body that allocates frame transients each turn needs the loop's +# per-iteration arena rewind -- the one a SELECT element already gets. The +# EACH lowerer built its ForStmt without it, so the arena grew for the whole +# loop and the checker rejected the body's iteration-scoped allocations +# (FRAME_NO_REWIND). + +FN summarize(items: []String, MUTABLE out: {String}String) RETURNS Int64 + REQUIRES out: LOCAL +-> +WITH POLYMORPHIC out AS MUTABLE view { + items |> EACH { + parts: []String = List[COPY _]; + first: String = COPY UNWRAP (parts[0_i64]); + view[COPY _] = first; + }; + RETURN view.length(); +} +END + +FN main() RETURNS Void -> + src: []String = List["a", "b"]; + MUTABLE m: {String}String = {}; + ASSERT summarize(src, &m) == 2_i64, "two"; + print("ok"); +END diff --git a/transpile-tests/913_rc_carrier_binding.clear b/transpile-tests/913_rc_carrier_binding.clear new file mode 100644 index 000000000..e6f557cda --- /dev/null +++ b/transpile-tests/913_rc_carrier_binding.clear @@ -0,0 +1,34 @@ +# Declaring an Rc/Arc binding from a plain value was broken four ways: the +# value was cast to the carrier type before the wrap (@as(Rc(T), plain)), the +# wrap was spelled against an optional payload (Rc(?T) instead of ?Rc(T)), a +# borrowed payload was moved into the handle instead of copied, and the +# handle got no release because the lifecycle plan read the payload's borrow +# provenance (ALLOC_WITHOUT_CLEANUP). + +PUB STRUCT Sig { name: String } +PUB UNION Holder { Sig: Sig, Count: Int64 } + +# The plain case: a declared carrier builds a handle around an owned value. +FN wrap_plain() RETURNS Int64 -> + s = Sig{ name: "plain" }; + MUTABLE out: Sig@multiowned = s; + RETURN out.name.length(); +END + +# The parser's shape: the payload is BORROWED out of a union, and the declared +# carrier is optional -- `?Sig@multiowned` is `?Rc(Sig)`, not `Rc(?Sig)`. +PUB FN unwrap(x: ?Holder) RETURNS ?Sig@multiowned -> + IF x? IS_A Sig AS sig THEN + MUTABLE out: ?Sig@multiowned = sig; + RETURN out; + END + RETURN NIL; +END + +FN main() RETURNS Void -> + ASSERT wrap_plain() == 5_i64, "plain carrier"; + h = Holder{ Sig: Sig{ name: "f" } }; + got: ?Sig@multiowned = unwrap(h); + ASSERT got EXISTS, "unwrapped"; + print("ok"); +END diff --git a/transpile-tests/914_symbol_valued_map_cleanup.clear b/transpile-tests/914_symbol_valued_map_cleanup.clear new file mode 100644 index 000000000..15531e9ae --- /dev/null +++ b/transpile-tests/914_symbol_valued_map_cleanup.clear @@ -0,0 +1,30 @@ +# Interned symbols are owned by the runtime intern table for the process +# lifetime. A `{String}String@symbol` map monomorphized to the owned-value +# StringMap freed every value at deinit (and on overwrite/delete), which is a +# misaligned free of intern-table storage. The map must select the +# interned-value representation, mirroring `@set` of symbols. + +PUB FN op_table() RETURNS {String}String@symbol -> + RETURN CAST({"+": :ADD, "-": :SUB} AS {String}String@symbol); +END + +PUB FN lookup(op_val: String) RETURNS String@symbol -> + RETURN (op_table()[op_val] OR_ELSE :UNKNOWN); +END + +FN main() RETURNS Void -> + known = lookup("+"); + unknown = lookup("?"); + ASSERT known == :ADD, "known op resolves"; + ASSERT unknown == :UNKNOWN, "unknown op falls back"; + + MUTABLE m: {String}String@symbol = {"a": :ADD}; + m["a"] = :SUB; + ASSERT UNWRAP (m["a"]) == :SUB, "overwrite keeps the interned value alive"; + &m.delete("a"); + ASSERT m.length() == 0_i64, "delete drops the entry without freeing the symbol"; + survivor = lookup("-"); + ASSERT survivor == :SUB, "the symbol survives the map that held it"; + + print("ok"); +END diff --git a/transpile-tests/915_runtime_interned_symbol_equality.clear b/transpile-tests/915_runtime_interned_symbol_equality.clear new file mode 100644 index 000000000..6eaa3f9af --- /dev/null +++ b/transpile-tests/915_runtime_interned_symbol_equality.clear @@ -0,0 +1,24 @@ +# A String@symbol has two representations that never share a pointer: the +# compiler-pooled rodata literal (`:ADD`) and the runtime intern-table handle +# `symbol(s)` returns for a string only known at runtime. Comparing them by +# pointer identity reported "not equal" for the same symbol. + +FN pick(index: Int64) RETURNS String -> + IF (index == 0_i64) THEN + RETURN "ADD"; + END + RETURN "SUB"; +END + +FN main() RETURNS Void -> + interned = symbol(pick(0_i64)); + other = symbol(pick(1_i64)); + + ASSERT interned == :ADD, "runtime-interned symbol equals the pooled literal"; + ASSERT other != :ADD, "a different symbol still compares unequal"; + ASSERT other == :SUB, "the second runtime-interned symbol matches its literal"; + ASSERT interned == symbol(pick(0_i64)), "interning is stable across calls"; + ASSERT :ADD == :ADD, "pooled literals still compare equal"; + + print("ok"); +END diff --git a/transpile-tests/916_nodrop_binding_not_owned.clear b/transpile-tests/916_nodrop_binding_not_owned.clear new file mode 100644 index 000000000..7dc1a8468 --- /dev/null +++ b/transpile-tests/916_nodrop_binding_not_owned.clear @@ -0,0 +1,41 @@ +# A binding whose type needs no drop -- an interned String@symbol -- owns no +# allocation, so it must not carry an AllocMark. The allocating-init plan +# stamped one anyway and then skipped the Cleanup, leaving a scope-local the +# checker could never see released: returning through a WITH scope failed with +# OWNERSHIP_UNVERIFIED_PATH. + +STRUCT BinaryOp { token: String, op: String@symbol } +UNION Locatable { BinaryOp: BinaryOp } +STRUCT Parser { pos: Int64, src: String } + +PUB FN op_table() RETURNS {String}String@symbol -> + RETURN CAST({"+": :ADD} AS {String}String@symbol); +END + +PUB FN parser__wrap(MUTABLE self: Parser, op_val: String) RETURNS !Locatable + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + view.pos = (view.pos + 1_i64); + op_sym = (op_table()[op_val] OR_ELSE symbol(op_val)); + RETURN Locatable{ BinaryOp: COPY BinaryOp{ token: COPY view.src, op: op_sym } }; +} +END + +FN main() RETURNS !Void -> + MUTABLE p = Parser{ pos: 0_i64, src: "t" }; + known = TRY parser__wrap(&p, "+"); + PARTIAL MATCH known START + Locatable.BinaryOp AS bin -> ASSERT bin.op == :ADD, "table hit keeps the pooled symbol";, + DEFAULT -> ASSERT FALSE, "expected a BinaryOp"; + END + + unknown = TRY parser__wrap(&p, "^"); + PARTIAL MATCH unknown START + Locatable.BinaryOp AS bin -> ASSERT bin.op == symbol("^"), "fallback interns the raw op";, + DEFAULT -> ASSERT FALSE, "expected a BinaryOp"; + END + ASSERT p.pos == 2_i64, "the WITH view mutation still lands on the receiver"; + + print("ok"); +END diff --git a/transpile-tests/917_map_literal_symbol_values.clear b/transpile-tests/917_map_literal_symbol_values.clear new file mode 100644 index 000000000..231fc0927 --- /dev/null +++ b/transpile-tests/917_map_literal_symbol_values.clear @@ -0,0 +1,37 @@ +# A map literal whose values are all symbols inferred {String}String, dropping +# @symbol. Its values then read as owned string slices: COPY deep-cloned them +# into the enclosing frame, and returning that value failed the escape check +# (FRAME_ALLOC_ESCAPES). List literals already preserve the element capability. + +STRUCT Fact { collection: String@symbol, soa: Bool } +STRUCT ListLit { token: String, options: ?Fact } +UNION Locatable { ListLit: ListLit } +STRUCT Parser { pos: Int64 } + +PUB FN parser__lit(MUTABLE self: Parser, name: String) RETURNS !?Locatable + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + view.pos = (view.pos + 1_i64); + kinds = {"List": :list, "Pool": :pool}; + collection = kinds[name]?; + MUTABLE node = ListLit{ token: COPY name, options: NIL }; + node.options = Fact{ collection: COPY collection, soa: FALSE }; + RETURN Locatable{ ListLit: COPY node }; +} +END + +FN main() RETURNS !Void -> + MUTABLE p = Parser{ pos: 0_i64 }; + got: ?Locatable = TRY parser__lit(&p, "Pool"); + PARTIAL MATCH UNWRAP got START + Locatable.ListLit AS lit -> ASSERT (UNWRAP lit.options).collection == :pool, "the symbol survives the escape";, + DEFAULT -> ASSERT FALSE, "expected a ListLit"; + END + + # An all-string map keeps the owned-value representation. + plain = {"a": "one", "b": "two"}; + ASSERT UNWRAP (plain["b"]) == "two", "string-valued maps still own their values"; + + print("ok"); +END diff --git a/transpile-tests/918_reassign_escaped_identifier.clear b/transpile-tests/918_reassign_escaped_identifier.clear new file mode 100644 index 000000000..bd3f09e32 --- /dev/null +++ b/transpile-tests/918_reassign_escaped_identifier.clear @@ -0,0 +1,32 @@ +# A binding whose name reads as a Zig primitive type (`f2`, `i8`, `u3`) is +# emitted as an escaped identifier, `@"f2"`. Templates that DERIVE an +# identifier from a binding name spliced the escape into the middle of the new +# name: `const __new_@"f2" = ...` for reassign temps, `var @"f2"_moved = false` +# for move guards. Neither is valid Zig. + +PUB FN pick(index: Int64) RETURNS String -> + IF (index == 0_i64) THEN + RETURN COPY "zero"; + END + RETURN COPY "other"; +END + +PUB FN consume(TAKES text: String) RETURNS Int64 -> + RETURN text.length(); +END + +FN main() RETURNS Void -> + MUTABLE f2: String = COPY "start"; + f2 = pick(0_i64); + ASSERT f2 == "zero", "reassignment through an escaped name"; + + MUTABLE i8: String = COPY "start"; + i8 = pick(1_i64); + ASSERT i8 == "other", "a second escaped name reassigns independently"; + + # A move guard: the owned value is given away, so cleanup must be skipped. + MUTABLE u3: String = pick(0_i64); + ASSERT consume(GIVE u3) == 4_i64, "move guard on an escaped name"; + + print("ok"); +END diff --git a/transpile-tests/919_zig_keyword_field_names.clear b/transpile-tests/919_zig_keyword_field_names.clear new file mode 100644 index 000000000..79d528d52 --- /dev/null +++ b/transpile-tests/919_zig_keyword_field_names.clear @@ -0,0 +1,20 @@ +# A struct field named after a Zig keyword needs the escaped spelling +# everywhere it appears: `comptime: bool` parses as a comptime field and +# `n.comptime` as the start of a comptime block. The declaration, accesses, +# struct-literal inits and the generated __clear_clone body all have to agree. +# The AST nodes being translated for self-hosting carry `comptime`, `fn`, +# `error` and `type` fields, so this is not hypothetical. + +STRUCT Node { comptime: Bool, fn: String, error: String@symbol } + +FN main() RETURNS Void -> + MUTABLE n = Node{ comptime: TRUE, fn: COPY "f", error: :NONE }; + n.comptime = FALSE; + n.fn = COPY "g"; + ASSERT !n.comptime, "keyword field write"; + ASSERT n.fn == "g", "keyword string field reassign with cleanup"; + ASSERT n.error == :NONE, "keyword symbol field"; + m = COPY n; + ASSERT m.fn == "g", "clone through keyword fields"; + print("ok"); +END diff --git a/transpile-tests/920_move_mark_before_terminator.clear b/transpile-tests/920_move_mark_before_terminator.clear new file mode 100644 index 000000000..936df6efc --- /dev/null +++ b/transpile-tests/920_move_mark_before_terminator.clear @@ -0,0 +1,39 @@ +# A MoveMark has to precede the move it guards. When the consuming node is a +# terminator -- `RETURN Wrapper{ field: owned_binding }` -- the mark was +# appended AFTER the return: the `_moved` guard was written too late to +# suppress the scope cleanup, and Zig rejected the statement as unreachable +# code. + +STRUCT VarDecl { name: String, value: Locatable } +STRUCT Lit { text: String } +UNION Locatable { Lit: Lit } +UNION Ret { VarDecl: VarDecl } +STRUCT Parser { pos: Int64 } + +PUB FN make(text: String) RETURNS !Locatable -> + RETURN Locatable{ Lit: COPY Lit{ text: COPY text } }; +END + +PUB FN parser__decl(MUTABLE self: Parser, name: String, has_value: Bool) RETURNS !Ret + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + view.pos = (view.pos + 1_i64); + IF has_value THEN + MUTABLE value = TRY make(name); + RETURN Ret{ VarDecl: COPY VarDecl{ name: COPY name, value: value } }; + END + MUTABLE fallback = TRY make("default"); + RETURN Ret{ VarDecl: COPY VarDecl{ name: COPY name, value: fallback } }; +} +END + +FN main() RETURNS !Void -> + MUTABLE p = Parser{ pos: 0_i64 }; + r = TRY parser__decl(&p, "x", TRUE); + PARTIAL MATCH r START + Ret.VarDecl AS d -> ASSERT d.name == "x", "decl";, + DEFAULT -> ASSERT FALSE, "expected VarDecl"; + END + print("ok"); +END diff --git a/transpile-tests/921_noreturn_panic_positions.clear b/transpile-tests/921_noreturn_panic_positions.clear new file mode 100644 index 000000000..7f137edce --- /dev/null +++ b/transpile-tests/921_noreturn_panic_positions.clear @@ -0,0 +1,49 @@ +# `panic(...)` is declared NoReturn, so it yields no value. Wrapping it in a +# value-producing construct emits code Zig rejects as unreachable: +# break :__match_1 @panic(...) (a MATCH arm) +# const __hoist_1 = @as(T, @panic(...)) (a hoisted RETURN value) +# return @as(T, @panic(...)) +# (x orelse @as(T, @panic(...))) (an OR_ELSE fallback) +# blk: { const __copy_src = @panic(...); (a COPY of that fallback) +# Each position has to emit the panic as the terminator it is. + +STRUCT GetField { name: String } +STRUCT Other { n: Int64 } +UNION Locatable { GetField: GetField, Other: Other } + +FN castLocatableToGetField(value: Locatable) RETURNS GetField -> + IF value IS_A GetField AS payload THEN + RETURN COPY payload; + END + RETURN panic("Invalid cast to GetField"); +END + +FN classify(kind: String@symbol) RETURNS Int64 -> + RETURN PARTIAL MATCH kind START + :a -> 1_i64, + :b -> 2_i64, + DEFAULT -> panic("unknown kind") + END; +END + +STRUCT Entry { n: Int64 } + +FN fetch(entries: {String}Entry, key: String) RETURNS Entry -> + RETURN (entries[key] OR_ELSE CAST(panic("missing hash key") AS Entry)); +END + +STRUCT Named { name: String } + +FN fetch_owned(entries: {String}Named, key: String) RETURNS Named -> + RETURN COPY (entries[key] OR_ELSE CAST(panic("missing hash key") AS Named)); +END + +FN main() RETURNS Void -> + g = castLocatableToGetField(Locatable{ GetField: GetField{ name: COPY "f" } }); + ASSERT g.name == "f", "the non-panicking cast path still returns its value"; + ASSERT classify(:a) == 1_i64, "first match arm"; + ASSERT classify(:b) == 2_i64, "second match arm"; + ASSERT fetch({"a": Entry{ n: 1_i64 }}, "a").n == 1_i64, "the OR_ELSE hit path still yields its value"; + ASSERT fetch_owned({"a": Named{ name: COPY "x" }}, "a").name == "x", "a COPY of that fallback still copies the hit"; + print("ok"); +END diff --git a/transpile-tests/922_union_return_owned.clear b/transpile-tests/922_union_return_owned.clear new file mode 100644 index 000000000..c77a2a8eb --- /dev/null +++ b/transpile-tests/922_union_return_owned.clear @@ -0,0 +1,38 @@ +# A function returning a union whose variants NAME structs owning heap fields +# returns an owned value. call_owned_return? asked variant_has_heap?, which +# only sees a bare heap pointer in the variant slot, and returned false -- so +# reassigning an optional from that call had no ownership operand at all +# (OWNERSHIP_CONSUMPTION_OPERAND_MISSING on the ReassignWithCleanup). + +STRUCT Lit { text: String } +STRUCT Other { n: Int64 } +UNION Locatable { Lit: Lit, Other: Other } +STRUCT Parser { pos: Int64 } + +PUB FN parser__expr(MUTABLE self: Parser) RETURNS !Locatable + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + view.pos = (view.pos + 1_i64); + RETURN Locatable{ Lit: COPY Lit{ text: COPY "e" } }; +} +END + +PUB FN parser__cap(MUTABLE self: Parser, has_guard: Bool) RETURNS !?Locatable + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS MUTABLE view { + MUTABLE guard_expr: ?Locatable = NIL; + IF has_guard THEN + guard_expr = TRY (parser__expr(&view)); + END + RETURN guard_expr; +} +END + +FN main() RETURNS !Void -> + MUTABLE p = Parser{ pos: 0_i64 }; + g: ?Locatable = TRY parser__cap(&p, TRUE); + ASSERT g != NIL, "guard parsed"; + print("ok"); +END diff --git a/transpile-tests/923_predicate_name_distinct.clear b/transpile-tests/923_predicate_name_distinct.clear new file mode 100644 index 000000000..3ceb485c2 --- /dev/null +++ b/transpile-tests/923_predicate_name_distinct.clear @@ -0,0 +1,21 @@ +# CLEAR distinguishes `raw` from `raw?` the way Ruby does. The Zig name +# mangling stripped the trailing mark, so the pair collapsed onto one +# identifier -- a duplicate declaration when both exist (ast/type.clear has +# type__raw and type__raw?), and a silent call to whichever won otherwise. + +STRUCT Shape { kind: String@symbol } + +PUB FN shape__raw(self: Shape) RETURNS String@symbol -> + RETURN self.kind; +END + +PUB FN shape__raw?(self: Shape) RETURNS Bool -> + RETURN (self.kind == :raw); +END + +FN main() RETURNS Void -> + s = Shape{ kind: :raw }; + ASSERT shape__raw(s) == :raw, "value accessor"; + ASSERT shape__raw?(s), "predicate"; + print("ok"); +END diff --git a/transpile-tests/924_placeholder_in_tuple_and_cast.clear b/transpile-tests/924_placeholder_in_tuple_and_cast.clear new file mode 100644 index 000000000..ebef98d1a --- /dev/null +++ b/transpile-tests/924_placeholder_in_tuple_and_cast.clear @@ -0,0 +1,31 @@ +# The pipeline placeholder rewriter dispatches on node type, and had no case +# for a tuple literal, a CAST, or a VarDecl initializer. A `_` nested inside +# any of them survived into the emitted Zig as the identifier `@"_"`, which +# does not exist -- the loop binds `__each_item`. + +STRUCT Bindings { entries: {String}Int64 } + +FN bindings__pairs(self: Bindings) RETURNS ![]Tuple + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS view { + MUTABLE pairs: []Tuple = List[]; + # `_` inside a tuple literal, and inside a CAST, both nested in an EACH body. + view.entries.keys() |> EACH { &pairs.append(CAST(Tuple{COPY _, COPY (view.entries[_] OR_ELSE 0_i64)} AS Tuple)); }; + # `_` inside a VarDecl initializer in an EACH body. + MUTABLE total = 0_i64; + view.entries.keys() |> EACH { + MUTABLE entry: Int64 = (view.entries[_] OR_ELSE 0_i64); + total = (total + entry); + }; + ASSERT total == 3_i64, "the VarDecl initializer saw each key"; + RETURN pairs; +} +END + +FN main() RETURNS !Void -> + b = Bindings{ entries: {"a": 1_i64, "b": 2_i64} }; + pairs = TRY bindings__pairs(b); + ASSERT pairs.length() == 2_i64, "both keys visited"; + print("ok"); +END diff --git a/transpile-tests/925_keyword_binding_and_unwrap_placeholder.clear b/transpile-tests/925_keyword_binding_and_unwrap_placeholder.clear new file mode 100644 index 000000000..98de089cb --- /dev/null +++ b/transpile-tests/925_keyword_binding_and_unwrap_placeholder.clear @@ -0,0 +1,33 @@ +# Two names that never reached their escaping/substitution pass: +# `IF x IS_A Ty AS type` bound the payload as a bare `type`, which Zig +# rejects as shadowing a primitive. +# `UNWRAP (map[_])` left the pipeline placeholder unsubstituted -- the +# rewriter had no case for AST::OptionalUnwrap. + +STRUCT Ty { n: Int64 } +UNION Value { Type: Ty, Num: Int64 } +STRUCT Bindings { entries: {String}Int64 } + +FN classify(x: Value) RETURNS Int64 -> + IF x IS_A Ty AS type THEN + RETURN type.n; + END + RETURN 0_i64; +END + +FN bindings__sum(self: Bindings) RETURNS Int64 + REQUIRES self: LOCAL +-> +WITH POLYMORPHIC self AS view { + MUTABLE total = 0_i64; + view.entries.keys() |> EACH { total = (total + UNWRAP (view.entries[_])); }; + RETURN total; +} +END + +FN main() RETURNS Void -> + ASSERT classify(Value{ Type: Ty{ n: 7_i64 } }) == 7_i64, "keyword-named binding"; + b = Bindings{ entries: {"a": 1_i64, "b": 2_i64} }; + ASSERT bindings__sum(b) == 3_i64, "placeholder under UNWRAP"; + print("ok"); +END diff --git a/transpile-tests/926_module_mutable_global.clear b/transpile-tests/926_module_mutable_global.clear new file mode 100644 index 000000000..68cfbc9de --- /dev/null +++ b/transpile-tests/926_module_mutable_global.clear @@ -0,0 +1,44 @@ +# A module-level `MUTABLE x = ...` is module state: `x = value` inside a +# function has to REASSIGN it. The routine-body scope was seeded empty, so the +# name resolved to nothing and the assignment silently declared a shadowing +# local -- the global never changed, and the emitted Zig redeclared the name: +# +# const next_id: i64 = CheatLib.intAdd(next_id, 1); + +MUTABLE next_id: Int64 = 11; +MUTABLE last_len: Int64 = 0_i64; +# A global that OWNS heap memory: it lives for the whole program, so the +# assignment targets the heap allocator and nothing drops it. +MUTABLE seen: ?[]Int64 = NIL; + +PUB FN take_id() RETURNS Int64 -> + MUTABLE id = COPY next_id; + next_id = (next_id + 1_i64); + RETURN id; +END + +PUB FN label(name: String) RETURNS Int64 -> + last_len = name.length(); + RETURN last_len; +END + +PUB FN remember(value: Int64) RETURNS !Int64 -> + IF seen == NIL THEN + seen = List[]; + END + IF seen EXISTS AS found THEN + RETURN found.length(); + END + RETURN 0_i64; +END + +FN main() RETURNS !Void -> + ASSERT take_id() == 11_i64, "first id"; + ASSERT take_id() == 12_i64, "the global advanced"; + ASSERT next_id == 13_i64, "and is visible from the outside"; + + ASSERT label("abc") == 3_i64, "second global reassigned"; + ASSERT last_len == 3_i64, "and kept the new value"; + ASSERT (TRY remember(1_i64)) == 0_i64, "an owned global is assignable"; + print("ok"); +END diff --git a/transpile-tests/927_local_shadows_parameter.clear b/transpile-tests/927_local_shadows_parameter.clear new file mode 100644 index 000000000..3835d6c71 --- /dev/null +++ b/transpile-tests/927_local_shadows_parameter.clear @@ -0,0 +1,38 @@ +# A local that shadows a parameter, or an IS_A payload binding, is legal CLEAR +# (and legal Ruby), but Zig rejects every shadowing. The disambiguating rename +# already existed for two same-named locals; parameters and payload bindings +# were not in the name table, so `MUTABLE type = COPY type` emitted a +# redeclaration. The suffix also has to be built from the CLEAR name -- +# `@"type"_L2` splices the escape into a new identifier. + +STRUCT Ty { n: Int64 } + +PUB FN widen(type: Ty) RETURNS Int64 -> + MUTABLE type = COPY type; + type.n = (type.n + 1_i64); + RETURN type.n; +END + +PUB FN pick(ft: Int64) RETURNS Int64 -> + MUTABLE ft = (ft * 2_i64); + RETURN ft; +END + +STRUCT Num { v: Int64 } +UNION Value { Ty: Ty, Num: Num } + +PUB FN widen_payload(x: Value) RETURNS Int64 -> + IF x IS_A Ty AS type THEN + MUTABLE type = COPY type; + type.n = (type.n + 1_i64); + RETURN type.n; + END + RETURN 0_i64; +END + +FN main() RETURNS Void -> + ASSERT widen(Ty{ n: 1_i64 }) == 2_i64, "shadowed param"; + ASSERT pick(3_i64) == 6_i64, "shadowed scalar param"; + ASSERT widen_payload(Value{ Ty: Ty{ n: 4_i64 } }) == 5_i64, "shadowed payload binding"; + print("ok"); +END diff --git a/transpile-tests/928_module_const_hoisted_parts.clear b/transpile-tests/928_module_const_hoisted_parts.clear new file mode 100644 index 000000000..1774fc179 --- /dev/null +++ b/transpile-tests/928_module_const_hoisted_parts.clear @@ -0,0 +1,39 @@ +# A module-level CONST table built from calls has each element hoisted to a +# temp. Those temps have nowhere to live at container scope, so the emitted +# Zig referenced undeclared names: +# +# const RULES = [2]Rule{ __tmp_3, __tmp_5 }; +# +# The initializer looks non-allocating by the time the CONST path sees it -- +# the allocation moved into the pending temps -- so it has to check for them +# and take the runtime-init prologue. + +STRUCT Rule { name: String, kind: Int64 } + +PUB FN rule(name: String, kind: Int64) RETURNS Rule -> + RETURN Rule{ name: COPY name, kind: kind }; +END + +CONST RULES: [2]Rule = [rule("a", 1_i64), rule("b", 2_i64)]; + +PUB FN first_name() RETURNS String -> + RETURN COPY RULES[0].name; +END + +PUB FN second_kind() RETURNS Int64 -> + RETURN RULES[1].kind; +END + +# A module-level MUTABLE global has the same container-scope problem. +MUTABLE defaults: [2]Rule = [rule("x", 9_i64), rule("y", 8_i64)]; + +PUB FN default_name() RETURNS String -> + RETURN COPY defaults[0].name; +END + +FN main() RETURNS Void -> + ASSERT first_name() == "a", "the table initialized before first use"; + ASSERT second_kind() == 2_i64, "and every element landed"; + ASSERT default_name() == "x", "a mutable global table initializes too"; + print("ok"); +END diff --git a/transpile-tests/929_each_body_ignores_item.clear b/transpile-tests/929_each_body_ignores_item.clear new file mode 100644 index 000000000..34242a4ef --- /dev/null +++ b/transpile-tests/929_each_body_ignores_item.clear @@ -0,0 +1,17 @@ +# An EACH body that ignores the item is ordinary CLEAR, but the list lowerer +# always named the loop capture `__each_item` -- and Zig rejects an unused +# capture. Vouch for it instead of predicting whether the body reads it: the +# usage scan does not see every position a `_` can appear in. + +FN main() RETURNS Void -> + MUTABLE total = 0_i64; + names: []String = ["a", "b"]; + kinds: []String = ["x", "y"]; + names |> EACH { + kinds |> EACH { + total = (total + 1_i64); + }; + }; + ASSERT total == 4_i64, "nested each"; + print("ok"); +END diff --git a/transpile-tests/930_nested_payload_binding_shadow.clear b/transpile-tests/930_nested_payload_binding_shadow.clear new file mode 100644 index 000000000..4aedede28 --- /dev/null +++ b/transpile-tests/930_nested_payload_binding_shadow.clear @@ -0,0 +1,28 @@ +# Two payload bindings of the same name nest: the inner one shadows the outer +# in CLEAR (and in Ruby), but Zig rejects every shadowing. The rename is keyed +# by the binding's DECLARATION -- a name-keyed map would keep pointing at the +# inner binding after the nested block ends, so the outer reference below the +# nested IF has to resolve back to the outer payload. + +STRUCT Ident { name: String } +STRUCT Call { name: String } +UNION Node { Ident: Ident, Call: Call } + +PUB FN describe(dst: Node, src: Node) RETURNS String -> + IF dst IS_A Ident AS item THEN + IF src IS_A Ident AS item THEN + RETURN COPY item.name; + END + RETURN COPY item.name; + END + RETURN COPY "none"; +END + +FN main() RETURNS Void -> + d = Node{ Ident: Ident{ name: COPY "a" } }; + s = Node{ Ident: Ident{ name: COPY "b" } }; + ASSERT describe(d, s) == "b", "inner payload binding wins"; + c = Node{ Call: Call{ name: COPY "c" } }; + ASSERT describe(d, c) == "a", "outer payload binding when inner misses"; + print("ok"); +END diff --git a/transpile-tests/933_map_literal_into_optional.clear b/transpile-tests/933_map_literal_into_optional.clear new file mode 100644 index 000000000..def6d0b23 --- /dev/null +++ b/transpile-tests/933_map_literal_into_optional.clear @@ -0,0 +1,24 @@ +# A map literal filling an OPTIONAL slot still builds a map. Taking the +# expected type as-is rendered the container as `?CheatLib.StringMap(V)`, +# which is not a struct literal Zig can initialize. + +STRUCT Entry { name: String } + +FN lookup(want: String) RETURNS ?{String}Entry -> + IF want == "none" THEN + RETURN NIL; + END + found: ?{String}Entry = {"a": Entry{ name: COPY "alpha" }}; + RETURN found; +END + +FN main() RETURNS Void -> + hit: ?{String}Entry = lookup("any"); + IF hit EXISTS AS table THEN + ASSERT (UNWRAP table["a"]).name == "alpha", "the optional map literal built a map"; + ELSE + ASSERT FALSE, "expected a table"; + END + ASSERT lookup("none") == NIL, "the NIL path still returns nothing"; + print("ok"); +END diff --git a/transpile-tests/934_interpolate_number_needs_tostring.clear b/transpile-tests/934_interpolate_number_needs_tostring.clear new file mode 100644 index 000000000..7f8646562 --- /dev/null +++ b/transpile-tests/934_interpolate_number_needs_tostring.clear @@ -0,0 +1,15 @@ +# `${n}` desugars to `$+`. A number has no bit-level coercion to a string, so +# a stamped String coercion could only emit `@as([]const u8, n)`. The rendering +# is explicit; interpolation of the rendered string is what works. + +STRUCT Tok { line: Int64 } + +FN main() RETURNS Void -> + t = Tok{ line: 42 }; + loc = " (line ${t.line.toString()})"; + ASSERT loc == " (line 42)", "the rendered number interpolates"; + n: Int64 = 7; + label = "n=" $+ n.toString(); + ASSERT label == "n=7", "explicit concat renders the same way"; + print("ok"); +END diff --git a/transpile-tests/935_copy_into_boxed_field.clear b/transpile-tests/935_copy_into_boxed_field.clear new file mode 100644 index 000000000..82cd20f8c --- /dev/null +++ b/transpile-tests/935_copy_into_boxed_field.clear @@ -0,0 +1,22 @@ +# COPY into a @boxed field duplicates the PAYLOAD: the box itself is made by +# the placement step. Typing the copy `?*T` told dupeValue the value was +# already a pointer. + +STRUCT Leaf { n: Int64 } +STRUCT Wrap { value: ?Node@boxed } +UNION Node { Leaf: Leaf, Wrap: Wrap } + +FN build(inner: Node) RETURNS Wrap -> + RETURN Wrap{ value: COPY inner }; +END + +FN build_empty() RETURNS Wrap -> + RETURN Wrap{ value: NIL }; +END + +FN main() RETURNS Void -> + w = build(Node{ Leaf: Leaf{ n: 7 } }); + ASSERT w.value != NIL, "the copied payload is boxed into the field"; + ASSERT build_empty().value == NIL, "an absent boxed optional stays absent"; + print("ok"); +END diff --git a/transpile-tests/936_optional_boxed_field.clear b/transpile-tests/936_optional_boxed_field.clear new file mode 100644 index 000000000..25a6858b9 --- /dev/null +++ b/transpile-tests/936_optional_boxed_field.clear @@ -0,0 +1,18 @@ +# `?T@boxed` is an optional POINTER: the box holds the payload and absence is +# the null pointer. Boxing the optional itself allocated a cell for `?T`, +# handed back `*?T`, and allocated even for NIL. + +STRUCT Leaf { n: Int64 } +STRUCT Wrap { value: ?Node@boxed } +UNION Node { Leaf: Leaf, Wrap: Wrap } + +FN build(inner: ?Node) RETURNS Wrap -> + RETURN Wrap{ value: COPY inner }; +END + +FN main() RETURNS Void -> + w = build(Node{ Leaf: Leaf{ n: 7 } }); + ASSERT w.value != NIL, "a present optional is boxed"; + ASSERT build(NIL).value == NIL, "an absent optional boxes nothing"; + print("ok"); +END diff --git a/transpile-tests/937_for_each_over_field_list.clear b/transpile-tests/937_for_each_over_field_list.clear new file mode 100644 index 000000000..e3110b8f0 --- /dev/null +++ b/transpile-tests/937_for_each_over_field_list.clear @@ -0,0 +1,23 @@ +# A `[]T@list` field is an ArrayList and iterates its `.items` just like a +# local. FOR picked its shape from the syntactic position instead, so a list +# reached through a field emitted `for (&list)`. + +STRUCT Item { n: Int64 } +STRUCT Sig { params: []Item } +STRUCT Fn { signature: Sig } + +FN total(f: Fn) RETURNS Int64 -> + MUTABLE total_n: Int64 = 0; + FOR param IN f.signature.params DO + total_n = total_n + param.n; + END + RETURN total_n; +END + +FN main() RETURNS Void -> + MUTABLE items: []Item = List[]; + &items.append(Item{ n: 2 }); + &items.append(Item{ n: 5 }); + ASSERT total(Fn{ signature: Sig{ params: items } }) == 7, "FOR walks a nested field list"; + print("ok"); +END diff --git a/transpile-tests/938_nested_map_literal.clear b/transpile-tests/938_nested_map_literal.clear new file mode 100644 index 000000000..b7c982247 --- /dev/null +++ b/transpile-tests/938_nested_map_literal.clear @@ -0,0 +1,19 @@ +# A nested map value builds against the outer map's VALUE type. Without it the +# inner literal guessed from its own items and the outer map stored the inner +# map's ENTRIES directly. + +UNION Entry { SymbolValue: String@symbol, StringValue: String } + +FN table() RETURNS {String@symbol}{String@symbol}?Entry -> + RETURN {:A: {:severity: Entry{ SymbolValue: :error }, :text: Entry{ StringValue: COPY "boom" }}}; +END + +FN main() RETURNS Void -> + t = table(); + IF t[:A] EXISTS AS row THEN + ASSERT (row[:severity]) != NIL, "the nested row is a map of its own"; + ELSE + ASSERT FALSE, "expected the A row"; + END + print("ok"); +END diff --git a/transpile-tests/939_empty_list_literal_destination_type.clear b/transpile-tests/939_empty_list_literal_destination_type.clear new file mode 100644 index 000000000..d1318d528 --- /dev/null +++ b/transpile-tests/939_empty_list_literal_destination_type.clear @@ -0,0 +1,28 @@ +STRUCT Item { n: Int64 } +STRUCT Bag { items: []Item } + +FN takes(label: String, items: []Item) RETURNS Int64 -> + MUTABLE mine: []Item = COPY items; + &mine.append(Item{ n: 2 }); + RETURN mine.length(); +END +FN empty_fallible() RETURNS ![]Item -> + RETURN List[]; +END +FN main() RETURNS Void -> + ASSERT takes("empty", List[]) == 1, "an empty literal argument takes the parameter element type"; + MUTABLE grown: []Item = TRY (empty_fallible()); + &grown.append(Item{ n: 3 }); + ASSERT grown.length() == 1, "an empty literal return takes the declared return element type"; + MUTABLE bag = Bag{ items: List[] }; + &bag.items.append(Item{ n: 4 }); + ASSERT bag.items.length() == 1, "an empty literal struct field takes the field element type"; + bag.items = List[]; + ASSERT bag.items.length() == 0, "an empty literal field assignment takes the field element type"; + MUTABLE shelves: {String}[]Item = {}; + shelves["a"] = List[]; + ASSERT (UNWRAP (shelves["a"])).length() == 0, "an empty literal map value takes the map value element type"; + MUTABLE declared: []Item = List[]; + &declared.append(Item{ n: 5 }); + ASSERT declared.length() == 1, "an empty literal declaration takes the declared element type"; +END diff --git a/transpile-tests/940_orelse_optional_fallback.clear b/transpile-tests/940_orelse_optional_fallback.clear new file mode 100644 index 000000000..f86178fb3 --- /dev/null +++ b/transpile-tests/940_orelse_optional_fallback.clear @@ -0,0 +1,36 @@ +STRUCT Rule { labels: []String } + +FN make_rule(label: String) RETURNS Rule -> + MUTABLE names: []String = List[]; + &names.append(COPY label); + RETURN Rule{ labels: names }; +END + +# `a OR_ELSE b` where b is ITSELF optional stays optional: typing the merge as +# the payload made placement copy a null as though it were present. +FN lookup(index: {String}Rule, first: String, second: String) RETURNS ?Rule -> + MUTABLE found: ?Rule = COPY index[first]; + found = COPY (found OR_ELSE index[second]); + RETURN COPY found; +END + +FN main() RETURNS Void -> + MUTABLE index: {String}Rule = {}; + index["a"] = make_rule("alpha"); + + IF lookup(index, "missing", "also_missing") EXISTS AS hit THEN + ASSERT FALSE, "neither lookup hits, so the merge stays absent"; + END + + IF lookup(index, "missing", "a") EXISTS AS second_hit THEN + ASSERT second_hit.labels.length() == 1, "an optional fallback that hits supplies the value"; + ELSE + ASSERT FALSE, "the fallback lookup should have hit"; + END + + IF lookup(index, "a", "missing") EXISTS AS first_hit THEN + ASSERT first_hit.labels.length() == 1, "a present left side still wins"; + ELSE + ASSERT FALSE, "the first lookup should have hit"; + END +END diff --git a/transpile-tests/941_tuple_return_promotes_list.clear b/transpile-tests/941_tuple_return_promotes_list.clear new file mode 100644 index 000000000..b680302fc --- /dev/null +++ b/transpile-tests/941_tuple_return_promotes_list.clear @@ -0,0 +1,28 @@ +STRUCT Leaf { names: []String } +UNION Kind { Leaf: Leaf } +STRUCT Item { kind: Kind@boxed } + +FN make_item() RETURNS Item -> + MUTABLE ns: []String = List[]; + &ns.append("x"); + RETURN Item{ kind: Kind{ Leaf: Leaf{ names: ns } } }; +END + +# A list returned inside a TUPLE escapes just as much as one returned bare. +# The hoisted tuple temp was promoted to the heap but its elements were not, +# so the list kept a frame allocation that outlived its frame. +FN collect(count: Int64, blk: FN() -> Elem) RETURNS Tuple -> + MUTABLE items: []Elem = List[]; + MUTABLE i = 0_i64; + WHILE (i < count) DO + &items.append(blk()); + i = (i + 1); + END + RETURN Tuple{count, items}; +END + +FN main() RETURNS Void -> + _, MUTABLE got = collect(2, make_item); + ASSERT got.length() == 2, "the tuple-returned list survives its frame"; + ASSERT (UNWRAP (got[0])).kind IS_A Leaf, "the escaped elements are intact"; +END diff --git a/transpile-tests/942_value_block_result_transfer.clear b/transpile-tests/942_value_block_result_transfer.clear new file mode 100644 index 000000000..406e2749a --- /dev/null +++ b/transpile-tests/942_value_block_result_transfer.clear @@ -0,0 +1,36 @@ +STRUCT Leaf { names: []String } +STRUCT Other { n: Int64 } +UNION Kind { Leaf: Leaf, Other: Other } +STRUCT Expr { kind: Kind@boxed } + +FN make(label: String) RETURNS Expr -> + MUTABLE ns: []String = List[]; + &ns.append(COPY label); + RETURN Expr{ kind: Kind{ Leaf: Leaf{ names: ns } } }; +END + +# A value block ending in `slot?` hands out its binding's payload while the +# binding still owns it. Whoever TAKES that result must claim the transfer, or +# the block's own cleanup frees the value on the way out and the receiver is +# left holding a dangling one. +FN from_raw(flag: Bool) RETURNS Expr -> + MUTABLE supplied: ?Expr = NIL; + MUTABLE parsed: Expr = (supplied OR_ELSE ({ MUTABLE marker = 0; MUTABLE slot: ?Expr = NIL; + IF flag THEN + slot = make("one"); + ELSE + slot = make("two"); + END + slot? })); + RETURN parsed; +END + +FN main() RETURNS Void -> + MUTABLE a = from_raw(TRUE); + MUTABLE seen = 0_i64; + IF a.kind IS_A Leaf AS leaf THEN + seen = leaf.names.length(); + ASSERT (UNWRAP (leaf.names[0])) == "one", "the escaped block result is intact, not freed"; + END + ASSERT seen == 1, "the block result kept its contents"; +END diff --git a/transpile-tests/943_destructured_tuple_element_heap.clear b/transpile-tests/943_destructured_tuple_element_heap.clear new file mode 100644 index 000000000..fe39bd684 --- /dev/null +++ b/transpile-tests/943_destructured_tuple_element_heap.clear @@ -0,0 +1,27 @@ +STRUCT Field { names: []String } + +FN make_field(label: String) RETURNS Field -> + MUTABLE ns: []String = List[]; + &ns.append(COPY label); + RETURN Field{ names: ns }; +END + +FN collect(count: Int64) RETURNS Tuple -> + MUTABLE items: []Field = List[]; + MUTABLE i = 0_i64; + WHILE (i < count) DO + &items.append(make_field("f")); + i = (i + 1); + END + RETURN Tuple{count, items}; +END + +# Destructuring into an ALREADY DECLARED binding: the target keeps the frame +# allocation its empty-literal initialiser chose, and it brings its own +# cleanup, so the temp the tuple arrived in must hand ownership over. +FN main() RETURNS Void -> + MUTABLE fields: []Field = List[]; + _, fields = collect(2); + ASSERT fields.length() == 2, "the destructured list survives its frame"; + ASSERT (UNWRAP ((UNWRAP (fields[0])).names[0])) == "f", "its elements are intact"; +END diff --git a/transpile-tests/944_unwrap_temp_is_a_view.clear b/transpile-tests/944_unwrap_temp_is_a_view.clear new file mode 100644 index 000000000..ce866d936 --- /dev/null +++ b/transpile-tests/944_unwrap_temp_is_a_view.clear @@ -0,0 +1,18 @@ +STRUCT Inner { names: []String } + +FN maybe(flag: Bool) RETURNS ?Inner -> + IF flag THEN + MUTABLE ns: []String = List[]; + &ns.append("x"); + RETURN Inner{ names: ns }; + END + RETURN NIL; +END + +# `tmp.?` is a VIEW of a temp that already owns the value. Hoisting it into a +# second owned binding gave the same heap parts two cleanups, which smashed the +# allocator free list rather than failing anywhere near the unwrap. +FN main() RETURNS Void -> + MUTABLE n = (UNWRAP (maybe(TRUE))).names.length(); + ASSERT n == 1, "the unwrapped call result is intact"; +END diff --git a/transpile-tests/945_const_map_lookup_is_borrow.clear b/transpile-tests/945_const_map_lookup_is_borrow.clear new file mode 100644 index 000000000..4ad90aca5 --- /dev/null +++ b/transpile-tests/945_const_map_lookup_is_borrow.clear @@ -0,0 +1,32 @@ +STRUCT Rule { action: String } + +FN build() RETURNS {String}Rule -> + RETURN {"a\x00b": Rule{ action: COPY "parse_stmt" }}; +END + +CONST IDX: {String}Rule = build(); + +FN make_key(left: String, right: String) RETURNS String -> + RETURN ((COPY left $+ "\x00") $+ COPY right); +END + +# A container lookup whose key needs a temp gets wrapped in a block, and owned +# placement treated that block's result as owned -- so it cleaned up a value +# the map still holds. The first call freed the map's own strings; every later +# lookup read (and re-freed) them. Only the COPY below belongs to the caller. +FN look() RETURNS String -> + MUTABLE found: ?Rule = IDX[make_key("a", "b")]; + IF found EXISTS AS rule THEN + RETURN COPY rule.action; + END + RETURN COPY "none"; +END + +FN main() RETURNS Void -> + first = look(); + second = look(); + third = look(); + ASSERT first == "parse_stmt", "the const map survives the first lookup"; + ASSERT second == "parse_stmt", "the const map survives a repeated lookup"; + ASSERT third == "parse_stmt", "the const map is not freed by its readers"; +END diff --git a/transpile-tests/946_union_without_owned_variant_drops.clear b/transpile-tests/946_union_without_owned_variant_drops.clear new file mode 100644 index 000000000..e79096594 --- /dev/null +++ b/transpile-tests/946_union_without_owned_variant_drops.clear @@ -0,0 +1,16 @@ +PUB UNION Dim { Int64Value: Int64, SymbolValue: String@symbol } + +STRUCT Holder { dims: []Dim } + +FN build() RETURNS Holder -> + RETURN Holder{ dims: [Dim{ SymbolValue: :LIST }, Dim{ Int64Value: 3 }] }; +END + +# A union whose variants own nothing got no `__clear_drop`, so cleanup fell +# through to representation-driven reflection -- which sees []const u8 and +# frees it, even though a String@symbol is rodata. The type has to state that +# it owns nothing rather than say nothing at all. +FN main() RETURNS Void -> + MUTABLE h = build(); + ASSERT h.dims.length() == 2, "both dimensions survive cleanup"; +END diff --git a/transpile-tests/947_struct_without_owned_field_drops.clear b/transpile-tests/947_struct_without_owned_field_drops.clear new file mode 100644 index 000000000..a3fb8cbd6 --- /dev/null +++ b/transpile-tests/947_struct_without_owned_field_drops.clear @@ -0,0 +1,16 @@ +STRUCT Tag { name: String@symbol, count: Int64 } +STRUCT Holder { tags: []Tag } + +FN build() RETURNS Holder -> + RETURN Holder{ tags: [Tag{ name: :alpha, count: 1 }, Tag{ name: :beta, count: 2 }] }; +END + +# A struct whose fields own nothing got no `__clear_drop`, so cleanup fell +# through to representation-driven reflection -- which sees []const u8 and frees +# it, though a String@symbol is rodata. The type has to state that it owns +# nothing rather than say nothing at all. Same contract as the union case in +# 946; a struct reaches it through a different lowering path. +FN main() RETURNS Void -> + MUTABLE h = build(); + ASSERT h.tags.length() == 2, "both tags survive cleanup"; +END diff --git a/transpile-tests/948_symbol_in_container_not_freed.clear b/transpile-tests/948_symbol_in_container_not_freed.clear new file mode 100644 index 000000000..6663cb8ca --- /dev/null +++ b/transpile-tests/948_symbol_in_container_not_freed.clear @@ -0,0 +1,20 @@ +# A `String@symbol` is interned: a literal points at rodata, and `symbol(str)` +# points into the Runtime's pool. Neither is the container's to free. But a +# symbol lowered to a plain []const u8 is indistinguishable from an owned +# String, so a collection of symbols freed its elements -- handing .rodata to +# the allocator. The type has to carry the fact. +FN main() RETURNS Void -> + MUTABLE holder: []String@symbol = []; + &holder.append(:alpha); + &holder.append(:beta); + + MUTABLE first: String@symbol = UNWRAP (holder[0]); + ASSERT first == :alpha, "a symbol read out of a list is intact"; + + MUTABLE again: String@symbol = UNWRAP (holder[0]); + ASSERT again == :alpha, "the first read did not free the list's element"; + + MUTABLE keyed: {String@symbol}Int64 = {}; + keyed[:alpha] = 1; + ASSERT (UNWRAP (keyed[:alpha])) == 1, "a symbol works as a map key"; +END diff --git a/transpile-tests/949_symbol_widens_to_string.clear b/transpile-tests/949_symbol_widens_to_string.clear new file mode 100644 index 000000000..863a6a499 --- /dev/null +++ b/transpile-tests/949_symbol_widens_to_string.clear @@ -0,0 +1,57 @@ +# A `String@symbol` is a distinct handle type, but in CLEAR it reads as a +# String. Every borrow position typed String must widen the handle to the +# bytes behind it -- and a TAKES position must receive an owned COPY, because +# the callee frees its parameter and interned bytes are nobody's to free. +FN borrow_len(s: String) RETURNS Int64 -> + RETURN s.length(); +END + +FN consume(TAKES s: String) RETURNS Int64 -> + RETURN s.length(); +END + +FN echo(tag: String@symbol) RETURNS String -> + RETURN CAST(tag AS String); +END + +FN main() RETURNS Void -> + MUTABLE tag: String@symbol = :alpha; + + ASSERT borrow_len(tag) == 5, "symbol borrows into a String param"; + # TAKES moves its argument (the annotator's rule for every non-COPY arg), + # so hand it a second binding of the same interned symbol. + MUTABLE doomed: String@symbol = :alpha; + ASSERT consume(doomed) == 5, "symbol into TAKES gets an owned copy"; + ASSERT echo(tag) == "alpha", "symbol returned through a String return"; + + print("tag is ${tag}"); + print(tag); + MUTABLE joined = ("x" $+ CAST(tag AS String)); + ASSERT joined == "xalpha", "symbol concatenates after CAST"; + MUTABLE inline_join = ("y" $+ tag); + ASSERT inline_join == "yalpha", "symbol concatenates directly"; + + # Registry calls widen too: a declared-String argument or receiver takes the + # bytes, and the `.len` fast path reads them. + ASSERT tag.length() == 5, "length on a symbol receiver"; + ASSERT tag.capitalize() == "Alpha", "string method on a symbol receiver"; + + MUTABLE maybe: ?String = NIL; + MUTABLE merged = "${(maybe OR_ELSE tag)}"; + ASSERT merged == "alpha", "symbol fallback merges into a String"; + + # The reverse merges hold their types: a symbol fallback into a symbol slot + # stays a handle, and a string LITERAL fallback narrows into one (rodata is + # immortal, so wrapping it orphans nothing). + MUTABLE none: ?String@symbol = NIL; + MUTABLE kept: String@symbol = (none OR_ELSE :beta); + ASSERT kept == :beta, "symbol fallback into a symbol merge stays a handle"; + MUTABLE narrowed: String@symbol = (none OR_ELSE "gamma"); + ASSERT narrowed == symbol("gamma"), "a literal fallback narrows into a symbol merge"; + + # A message-less ASSERT lowers to Zig's expectEqualStrings instead of + # CheatLib.assert, and that helper takes []const u8 -- one more String + # coercion boundary the handle has to widen at. + ASSERT tag == :alpha; + ASSERT echo(tag) == "alpha"; +END diff --git a/transpile-tests/950_union_inline_variant_releases_shared.clear b/transpile-tests/950_union_inline_variant_releases_shared.clear new file mode 100644 index 000000000..bfaec412f --- /dev/null +++ b/transpile-tests/950_union_inline_variant_releases_shared.clear @@ -0,0 +1,20 @@ +STRUCT Item { value: Int64 } + +UNION Holder { Wrapped { item: Item@shared }, Empty } + +# An inline-struct variant decided whether to drop its payload from the +# variant's `deinit_entries`, not from the lifecycle registry, so a payload +# that owns something only through a capability -- an Arc field -- got an empty +# drop arm. Since `__clear_drop` now always exists and cleanup consults it +# BEFORE representation-driven reflection, that empty arm was the whole +# contract: the Arc was never released, and its control block and payload +# leaked. +FN main() RETURNS Void -> + MUTABLE seen = 0_i64; + holder: Holder = Holder.Wrapped{ item: Item{ value: 7 } @shared }; + MATCH holder START + Holder.Wrapped AS w -> seen = w.item.value;, + Holder.Empty -> seen = 0_i64; + END + ASSERT seen == 7_i64, "the shared payload is readable"; +END diff --git a/transpile-tests/module-integration/packages/geometry/src/lib.clear b/transpile-tests/module-integration/packages/geometry/src/lib.clear index 531ed3049..3575060ba 100644 --- a/transpile-tests/module-integration/packages/geometry/src/lib.clear +++ b/transpile-tests/module-integration/packages/geometry/src/lib.clear @@ -3,3 +3,11 @@ REQUIRE "pkg:math"; PUB FN distance_sq(x: Number, y: Number) RETURNS Number -> RETURN add(square(x), square(y)); END + +PUB FN shape_size(shape: Shape) RETURNS Int64 -> + PARTIAL MATCH shape START + Shape.Circle AS payload -> RETURN payload.radius;, + Shape.Square AS payload -> RETURN payload.side; + END + RETURN 0_i64; +END diff --git a/transpile-tests/module-integration/packages/math/src/lib.clear b/transpile-tests/module-integration/packages/math/src/lib.clear index ac133a2f7..bc13f940d 100644 --- a/transpile-tests/module-integration/packages/math/src/lib.clear +++ b/transpile-tests/module-integration/packages/math/src/lib.clear @@ -9,3 +9,28 @@ END PUB FN square(x: Number) RETURNS Number -> RETURN multiply(x, x); END + +PUB STRUCT Tag { name: String, weight: Int64 } + +PUB FN tag(name: String, weight: Int64) RETURNS Tag -> + RETURN Tag{ name: COPY name, weight: weight }; +END + +# A module-scope table built from allocating calls: its hoisted temps must not +# drain into the next function's body. +tags: [2]Tag = [tag("alpha", 1), tag("beta", 2)]; + +PUB FN tag_weight(index: Int64) RETURNS Int64 -> + RETURN tags[index].weight; +END + +# An imported union: MATCH dispatch in a CONSUMING package has to see this +# schema, or `Shape.Circle AS payload` lowers to a tag equality test and never +# binds the payload. +PUB STRUCT Circle { radius: Int64 } +PUB STRUCT Square { side: Int64 } +PUB UNION Shape { Circle: Circle, Square: Square } + +PUB FN circle(radius: Int64) RETURNS Shape -> + RETURN Shape{ Circle: Circle{ radius: radius } }; +END diff --git a/transpile-tests/module-integration/src/main.clear b/transpile-tests/module-integration/src/main.clear index 8ffc958d3..6e5016e27 100644 --- a/transpile-tests/module-integration/src/main.clear +++ b/transpile-tests/module-integration/src/main.clear @@ -13,4 +13,10 @@ FN main() RETURNS Void -> dist = distance_sq(3, 4); ASSERT dist == 25; + + ASSERT tag_weight(1) == 2; + + # The union crosses two package boundaries: declared in math, matched in + # geometry, constructed here. + ASSERT shape_size(circle(7)) == 7; END diff --git a/zig/build.zig b/zig/build.zig index 09658dfdc..d06b19da8 100644 --- a/zig/build.zig +++ b/zig/build.zig @@ -719,6 +719,7 @@ pub fn build(b: *std.Build) void { "scheduler-benchmark-test.zig", "parking-lot-benchmark-test.zig", "versioned-benchmark-test.zig", + "symbol-intern-benchmark-test.zig", "experimental/freeze_bench.zig", }; diff --git a/zig/lib/data-structures-test.zig b/zig/lib/data-structures-test.zig index 62347fd42..78b1f9bcf 100644 --- a/zig/lib/data-structures-test.zig +++ b/zig/lib/data-structures-test.zig @@ -443,39 +443,6 @@ test "typed-key maps support structural keys and own their key data" { try std.testing.expectEqual(@as(i64, 0), CheatLib.numericMapCount(Key, i64, map)); } -test "InternedStringSet never frees interned elements (insert/dup/remove/deinit)" { - const allocator = std.testing.allocator; - var set: CheatLib.InternedStringSet() = .{}; - defer set.deinit(allocator); - - // Rodata literals stand in for intern-table symbols: any free would - // crash or corrupt, and std.testing.allocator would flag a non-owned - // pointer immediately. - try set.insert(allocator, "alpha"); - try set.insert(allocator, "beta"); - try set.insert(allocator, "alpha"); // duplicate: must NOT free - try std.testing.expectEqual(@as(i64, 2), set.length()); - try std.testing.expect(set.contains("alpha")); - - set.remove(allocator, "beta"); // must NOT free - try std.testing.expectEqual(@as(i64, 1), set.length()); -} - -test "InternedStringSet cleanup and dupeValue reuse element pointers" { - const allocator = std.testing.allocator; - var set: CheatLib.InternedStringSet() = .{}; - try set.insert(allocator, "gamma"); - try set.insert(allocator, "delta"); - - var copy = try CheatLib.dupeValue(CheatLib.InternedStringSet(), set, allocator); - try std.testing.expectEqual(@as(i64, 2), copy.length()); - try std.testing.expect(copy.contains("gamma")); - - // Generic cleanup path must only free the backing maps. - CheatLib.cleanup(CheatLib.InternedStringSet(), allocator, ©); - CheatLib.cleanup(CheatLib.InternedStringSet(), allocator, &set); -} - test "owned-string Set still frees duplicates and elements at deinit" { const allocator = std.testing.allocator; var set: CheatLib.Set([]const u8) = .{}; @@ -539,3 +506,17 @@ test "sharded getPtr reaches an aggregate payload without copying it" { try std.testing.expectEqual(@as(usize, 1), observed.edges.items.len); try std.testing.expectEqual(@as(i64, 5), observed.edges.items[0]); } +test "owned-value StringMap still frees replaced and removed values" { + const allocator = std.testing.allocator; + var map: CheatLib.StringMap([]const u8) = .{}; + map.alloc = allocator; + defer map.deinit(allocator, allocator); + + try map.put(allocator, allocator, "k", try allocator.dupe(u8, "first")); + try map.put(allocator, allocator, "k", try allocator.dupe(u8, "second")); // frees "first" + try map.put(allocator, allocator, "j", try allocator.dupe(u8, "third")); + try std.testing.expectEqual(@as(i64, 2), map.count()); + + map.remove(allocator, "j"); // frees "third" + try std.testing.expectEqual(@as(i64, 1), map.count()); +} diff --git a/zig/lib/data-structures.zig b/zig/lib/data-structures.zig index 28f876b0b..e1845d216 100644 --- a/zig/lib/data-structures.zig +++ b/zig/lib/data-structures.zig @@ -53,7 +53,11 @@ pub fn bind(comptime deps: type) type { const return_type = get_info.return_type.?; return struct { pub const StorageType = Storage; - pub const Key = get_info.params[1].type.?; + // `get` takes `anytype` so a Symbol key can normalize to bytes + // at the boundary, which leaves its param type null. A map that + // states its own key type is the authority; fall back to + // reflection for those that do not. + pub const Key = if (@hasDecl(Storage, "Key")) Storage.Key else get_info.params[1].type.?; pub const Value = @typeInfo(return_type).optional.child; }; } @@ -185,6 +189,19 @@ pub fn bind(comptime deps: type) type { // HashMap@sharded(N) at the declaration site is a one-line change that // doesn't ripple through function signatures. // ----------------------------------------------------------------------- + /// Map keys are bytes. A `String@symbol` key arrives as a Symbol handle -- + /// same bytes, different type -- so normalize at the boundary rather than + /// making every caller unwrap. One implementation: CheatLib.bytesOf, + /// threaded through the bind deps because this file cannot import it. + pub inline fn keyBytes(key: anytype) []const u8 { + return deps.bytesOf(key); + } + + // A map of interned symbols needs no special variant: CheatLib.Symbol's + // drop is a no-op, so the ordinary owned-value map leaves intern-table + // storage alone. InternedValueStringMap/InternedStringSet existed because + // a symbol was spelled []const u8 and the containers could not tell it + // from an owned String. pub fn StringMap(comptime V: type) type { return struct { const Self = @This(); @@ -198,7 +215,11 @@ pub fn bind(comptime deps: type) type { /// TAKES ownership of value. Strings are duped (may be rodata/frame). /// TAKES ownership of value. No implicit copies. Caller must /// ensure all data (including strings) is heap-owned. - pub fn put(self: *Self, key_alloc: std.mem.Allocator, bucket_alloc: std.mem.Allocator, key: []const u8, value: V) !void { + /// Byte-keyed: a `String@symbol` lookup normalizes through keyBytes. + pub const Key = []const u8; + + pub fn put(self: *Self, key_alloc: std.mem.Allocator, bucket_alloc: std.mem.Allocator, key_in: anytype, value: V) !void { + const key = keyBytes(key_in); _ = key_alloc; _ = bucket_alloc; const stored_value = value; @@ -212,15 +233,18 @@ pub fn bind(comptime deps: type) type { } - pub fn get(self: anytype, key: []const u8) ?V { + pub fn get(self: anytype, key_in: anytype) ?V { + const key = keyBytes(key_in); return self.inner.get(key); } - pub fn contains(self: anytype, key: []const u8) bool { + pub fn contains(self: anytype, key_in: anytype) bool { + const key = keyBytes(key_in); return self.inner.contains(key); } - pub fn remove(self: *Self, key_alloc: std.mem.Allocator, key: []const u8) void { + pub fn remove(self: *Self, key_alloc: std.mem.Allocator, key_in: anytype) void { + const key = keyBytes(key_in); _ = key_alloc; if (self.inner.fetchRemove(key)) |kv| { self.alloc.free(kv.key); @@ -247,7 +271,8 @@ pub fn bind(comptime deps: type) type { /// Free heap-allocated payloads inside tagged union values. // Delegate to inner for code that still uses raw HashMap API - pub fn getPtr(self: *Self, key: []const u8) ?*V { + pub fn getPtr(self: *Self, key_in: anytype) ?*V { + const key = keyBytes(key_in); return self.inner.getPtr(key); } @@ -2443,17 +2468,10 @@ pub fn bind(comptime deps: type) type { // AutoHashMapUnmanaged(T, void) for other types. // ----------------------------------------------------------------------- pub fn Set(comptime T: type) type { - return SetImpl(T, true); + return SetImpl(T); } - /// Set of interned strings (CLEAR `[Set]String@symbol`): elements are - /// intern-table/rodata handles the set never owns. No frees on - /// duplicate insert, remove, or deinit; COPY reuses element pointers. - pub fn InternedStringSet() type { - return SetImpl([]const u8, false); - } - - fn SetImpl(comptime T: type, comptime owned_elements: bool) type { + fn SetImpl(comptime T: type) type { const is_string = T == []const u8; const Context = struct { pub fn hash(_: @This(), key: T) u64 { @@ -2480,7 +2498,6 @@ pub fn bind(comptime deps: type) type { std.HashMapUnmanaged(T, void, Context, std.hash_map.default_max_load_percentage); return struct { const Self = @This(); - pub const interned_elements = !owned_elements; inner: Map = .{}, pub fn initCapacity(alloc: std.mem.Allocator, capacity: u32) !Self { @@ -2492,7 +2509,7 @@ pub fn bind(comptime deps: type) type { pub fn insert(self: *Self, alloc: std.mem.Allocator, value: T) !void { if (is_string) { if (self.inner.contains(value)) { - if (owned_elements) alloc.free(value); + alloc.free(value); } else { try self.inner.put(alloc, value, {}); } @@ -2513,7 +2530,7 @@ pub fn bind(comptime deps: type) type { pub fn remove(self: *Self, alloc: std.mem.Allocator, value: T) void { if (is_string) { if (self.inner.fetchRemove(value)) |kv| { - if (owned_elements) alloc.free(kv.key); + alloc.free(kv.key); } } else { if (self.inner.fetchRemove(value)) |kv| { @@ -2537,10 +2554,8 @@ pub fn bind(comptime deps: type) type { pub fn deinit(self: *Self, alloc: std.mem.Allocator) void { if (is_string) { - if (owned_elements) { - var it = self.inner.keyIterator(); - while (it.next()) |key_ptr| alloc.free(key_ptr.*); - } + var it = self.inner.keyIterator(); + while (it.next()) |key_ptr| alloc.free(key_ptr.*); } else if (comptime needsCleanup(T)) { var it = self.inner.keyIterator(); while (it.next()) |key_ptr| cleanup(T, alloc, key_ptr); diff --git a/zig/runtime/arena-mode-test.zig b/zig/runtime/arena-mode-test.zig index 3bce8dd72..670675d8b 100644 --- a/zig/runtime/arena-mode-test.zig +++ b/zig/runtime/arena-mode-test.zig @@ -174,3 +174,37 @@ test "heapAlloc uses pinned local allocator when set" { // They should be different allocators try std.testing.expect(global.ptr != pinned.ptr); } + +test "owns: current storage, retired storage, and foreign pointers" { + // `owns` backs the frame allocator's foreign-free check, so its three + // answers are load-bearing: memory the arena currently holds is owned, + // memory it handed out and has since reclaimed is STILL owned (a no-op + // cleanup may run after the rewind that trimmed the block -- that + // sequence is legitimate), and anything else is a foreign free. + var static_buf: [512]u8 = undefined; + var arena = CheatArena.init(std.testing.allocator, &static_buf); + defer arena.deinit(); + + // Static-block storage is owned. + const in_static = arena.alloc(64, 8, 0).?; + try std.testing.expect(arena.owns(in_static)); + + // Overflow-block storage is owned while live... + const mark = arena.getMark(); + const spilled = arena.alloc(8 * 1024, 8, 0).?; + try std.testing.expect(arena.owns(spilled)); + + // ...and stays owned after the rewind retires its block: the envelope + // remembers reclaimed address space precisely so late no-op cleanups are + // not misread as foreign. + arena.rewind(mark); + try std.testing.expect(arena.owns(spilled)); + + // Memory this arena never handed out is foreign -- rodata and the heap + // are the callers that must be rejected. + const rodata: []const u8 = "not the arena's to free"; + try std.testing.expect(!arena.owns(@constCast(rodata.ptr))); + const heap = try std.testing.allocator.alloc(u8, 32); + defer std.testing.allocator.free(heap); + try std.testing.expect(!arena.owns(heap.ptr)); +} diff --git a/zig/runtime/cleanup-test.zig b/zig/runtime/cleanup-test.zig index 72247f62e..b3c7d3c9c 100644 --- a/zig/runtime/cleanup-test.zig +++ b/zig/runtime/cleanup-test.zig @@ -595,6 +595,25 @@ test "dupeUnionValue deep-copies string variant independently" { CheatLib.cleanup(TestValue, alloc, ©_mut); } +test "dupeValue copies the payload of an already-narrowed optional source" { + const alloc = std.testing.allocator; + + var items = std.ArrayListUnmanaged([]const u8).empty; + try items.append(alloc, try alloc.dupe(u8, "a")); + const narrowed: ?StringListValue = StringListValue{ .Items = items }; + + // The destination type is concrete; the source arrives optional because the + // caller narrowed it. Copying must reach the payload's clone glue. + const copied = try CheatLib.dupeValue(StringListValue, narrowed, alloc); + try std.testing.expectEqual(@as(usize, 1), copied.Items.items.len); + try std.testing.expectEqualStrings("a", copied.Items.items[0]); + + var orig_mut = narrowed.?; + CheatLib.cleanup(StringListValue, alloc, &orig_mut); + var copy_mut = copied; + CheatLib.cleanup(StringListValue, alloc, ©_mut); +} + test "dupeValue deep-copies union ArrayList string payload elements independently" { const alloc = std.testing.allocator; diff --git a/zig/runtime/fiber-memory.zig b/zig/runtime/fiber-memory.zig index 2f1d30e3f..7bff0551b 100644 --- a/zig/runtime/fiber-memory.zig +++ b/zig/runtime/fiber-memory.zig @@ -88,7 +88,11 @@ pub const MICRO_STACK_SIZE: usize = 4 * 1024; // 4 KB pub const STANDARD_STACK_SIZE: usize = 16 * 1024; // 16 KB (default) pub const LARGE_STACK_SIZE: usize = 64 * 1024; // 64 KB pub const XL_STACK_SIZE: usize = 256 * 1024; // 256 KB -pub const HUGE_STACK_SIZE: usize = 4 * 1024 * 1024; // 4 MB service stack +// Heap-allocated on demand, so the cost is address space and the pages a +// fiber actually touches. The self-hosted CLEAR parser needs well past 4 MB: +// recursive descent alone reaches ~2.5 MB before a Locatable clone, whose own +// Debug frame is ~2 MB (a union's clone reserves one temp per variant). +pub const HUGE_STACK_SIZE: usize = 32 * 1024 * 1024; // 32 MB service stack // Typed array aliases — each SlabAllocator is parameterized by a fixed-size type. const MicroArray = [MICRO_STACK_SIZE]u8; diff --git a/zig/runtime/frame.zig b/zig/runtime/frame.zig index d05719203..9ecff45c3 100644 --- a/zig/runtime/frame.zig +++ b/zig/runtime/frame.zig @@ -40,6 +40,26 @@ pub fn CheatArenaType(comptime debug_mode: bool) type { // An optional pre-allocated buffer (e.g. the 4KB Frame) static_block: []u8 = &[_]u8{}, + // Safety builds only: address ranges this arena has handed out and + // since reclaimed. `owns` has to answer "was this ever mine?", not + // "is it mine right now" -- an arena rewinds and trims blocks while + // no-op cleanups for values inside them are still pending, and that + // sequence is legitimate. + // Fixed and inline: an arena that is trimmed but never deinit'd (a + // detached fiber's) would leak a growable list, and blocks grow + // geometrically so the count stays small. If it ever wraps we stop + // answering, rather than answer wrongly. + // An envelope, not exact ranges: the check exists to catch frees of + // .rodata, interned symbols, and container-owned storage, none of + // which lives anywhere near this arena's heap blocks, so a coarse + // [lo, hi) loses essentially no real detection. What it buys is + // decisive -- a fixed 16 bytes. Runtime embeds this arena by value and + // a fiber keeps its Runtime ON THE FIBER STACK; an exact-range table + // grew that by 2 KB and overflowed 16 KB fiber stacks straight into + // the neighboring stack slab. + retired_lo: if (is_debug) usize else void = if (is_debug) std.math.maxInt(usize) else {}, + retired_hi: if (is_debug) usize else void = if (is_debug) 0 else {}, + pub fn init(child_allocator: std.mem.Allocator, static_block: []u8) Self { return .{ .blocks = .empty, @@ -49,6 +69,38 @@ pub fn CheatArenaType(comptime debug_mode: bool) type { }; } + /// Did this pointer come from this arena? Used by the frame allocator's + /// free path in safety builds: the arena frees nothing, so without this + /// a cleanup aimed at .rodata, the heap, or another fiber's arena is + /// silently accepted and the mistake only surfaces as a crash somewhere + /// unrelated -- or never. + fn retire(self: *Self, slice: []u8) void { + if (!is_debug) return; + const base = @intFromPtr(slice.ptr); + self.retired_lo = @min(self.retired_lo, base); + self.retired_hi = @max(self.retired_hi, base + slice.len); + } + + pub fn owns(self: *Self, ptr: [*]u8) bool { + const addr = @intFromPtr(ptr); + if (is_debug) { + if (addr >= self.retired_lo and addr < self.retired_hi) return true; + } + if (self.static_block.len > 0) { + const base = @intFromPtr(self.static_block.ptr); + if (addr >= base and addr < base + self.static_block.len) return true; + } + for (self.blocks.items) |block| { + const base = @intFromPtr(block.ptr); + if (addr >= base and addr < base + block.len) return true; + } + for (self.large_objects.items) |obj| { + const base = @intFromPtr(obj.slice.ptr); + if (addr >= base and addr < base + obj.slice.len) return true; + } + return false; + } + pub fn deinit(self: *Self) void { for (self.blocks.items) |block| { // rawFree requires the alignment we allocated with. @@ -217,6 +269,7 @@ pub fn CheatArenaType(comptime debug_mode: bool) type { if (!debug_mode) { while (self.large_objects.items.len > mark.large_obj_count) { const popped = self.large_objects.pop().?; + self.retire(popped.slice); self.child_allocator.rawFree(popped.slice, popped.alignment, @returnAddress()); } } @@ -240,6 +293,7 @@ pub fn CheatArenaType(comptime debug_mode: bool) type { _ = large_align; while (self.large_objects.items.len > mark.large_obj_count) { const popped = self.large_objects.pop().?; + self.retire(popped.slice); self.child_allocator.rawFree(popped.slice, popped.alignment, @returnAddress()); } } @@ -263,12 +317,14 @@ pub fn CheatArenaType(comptime debug_mode: bool) type { // Free Large Objects while (self.large_objects.items.len > mark.large_obj_count) { const popped = self.large_objects.pop().?; + self.retire(popped.slice); self.child_allocator.rawFree(popped.slice, popped.alignment, @returnAddress()); } // Trim Blocks while (self.blocks.items.len > keep_count) { const popped = self.blocks.pop().?; + self.retire(popped); self.child_allocator.rawFree(popped, large_align, @returnAddress()); } } diff --git a/zig/runtime/runtime-header.zig b/zig/runtime/runtime-header.zig index 6256d84d2..27c4993de 100644 --- a/zig/runtime/runtime-header.zig +++ b/zig/runtime/runtime-header.zig @@ -1111,6 +1111,11 @@ pub const CheatLib = struct { // ========================================================================= const DataStructures = @import("../lib/data-structures.zig").bind(struct { + /// The bytes behind a String or a Symbol; identity for anything + /// already byte-shaped. Container key normalization delegates here so + /// the nominal Symbol test has exactly one implementation. + pub const bytesOf = CheatLib.bytesOf; + pub fn cleanup(comptime T: type, alloc: std.mem.Allocator, cptr: *const T) void { CheatLib.cleanup(T, alloc, cptr); } @@ -1614,7 +1619,6 @@ pub const CheatLib = struct { pub const ShardedPool = DataStructures.ShardedPool; pub const ShardedList = DataStructures.ShardedList; pub const Set = DataStructures.Set; - pub const InternedStringSet = DataStructures.InternedStringSet; pub const PartitionedStringMap = DataStructures.PartitionedStringMap; pub const PartitionedNumericMap = DataStructures.PartitionedNumericMap; pub const ShardedStringMap = DataStructures.ShardedStringMap; @@ -2277,8 +2281,8 @@ pub const CheatLib = struct { // schedulers/threads. // String Equality (Content check) - pub fn strEql(s1: []const u8, s2: []const u8) bool { - return std.mem.eql(u8, s1, s2); + pub fn strEql(s1: anytype, s2: anytype) bool { + return std.mem.eql(u8, bytesOf(s1), bytesOf(s2)); } // Lexicographic string comparison. Returns -1, 0, or 1. @@ -2296,6 +2300,8 @@ pub const CheatLib = struct { const T = @TypeOf(a); const info = @typeInfo(T); + if (T == Symbol) return a.eqlSymbol(b); + // For slices (like strings), use mem.eql if (info == .pointer and info.pointer.size == .slice) { return std.mem.eql(info.pointer.child, a, b); @@ -2767,6 +2773,17 @@ pub const CheatLib = struct { return buf; } + // capitalize(str) -> new string with the first ASCII byte uppercased and + // the rest lowered, matching Ruby's String#capitalize. + pub fn stringCapitalize(allocator: std.mem.Allocator, str: []const u8) ![]const u8 { + Runtime.profileAlloc(str.len); + const buf = try allocator.alloc(u8, str.len); + for (str, 0..) |c, idx| { + buf[idx] = if (idx == 0) std.ascii.toUpper(c) else std.ascii.toLower(c); + } + return buf; + } + // shell pub fn shell(allocator: std.mem.Allocator, cmd: []const u8) ![]const u8 { @@ -3757,6 +3774,53 @@ pub const CheatLib = struct { } } + /// An interned string: a `:literal` points at .rodata, `symbol(str)` points + /// into the Runtime's pool. Either way the bytes outlive every handle and + /// belong to nobody, so a Symbol is Copy and dropping one is a no-op. + /// + /// Keeping it distinct from []const u8 is the whole point. The two are + /// identical in representation, so while a symbol was spelled []const u8 + /// nothing downstream could tell it from an owned String -- a collection of + /// symbols freed its elements and handed .rodata to the allocator. Only the + /// type can carry that fact. + pub const Symbol = struct { + bytes: []const u8, + + pub fn __clear_drop(self: *@This(), alloc: std.mem.Allocator) void { + _ = self; + _ = alloc; + } + + pub fn __clear_clone(self: @This(), alloc: std.mem.Allocator) !@This() { + _ = alloc; + return self; + } + + /// Interning makes identity pointer identity, so that is the fast path. + /// It is not sufficient alone: `:alpha` in two modules is two rodata + /// constants, and a pooled symbol is a third address for the same name. + pub fn eqlSymbol(self: @This(), other: @This()) bool { + if (self.bytes.ptr == other.bytes.ptr) return true; + return std.mem.eql(u8, self.bytes, other.bytes); + } + }; + + /// The bytes behind a String or a Symbol. Widening a symbol to a string is + /// always safe -- it is a borrow of interned storage -- so the conversion + /// resolves at comptime instead of at every call site. + pub inline fn bytesOf(value: anytype) []const u8 { + const T = @TypeOf(value); + if (comptime T == Symbol) return value.bytes; + return value; + } + + /// Wrap interned bytes as a Symbol. A helper rather than a struct literal + /// because the stdlib zig templates substitute `{0}`-style holes and do not + /// escape braces. + pub inline fn symbolOf(bytes: []const u8) Symbol { + return .{ .bytes = bytes }; + } + /// Unified comptime cleanup for any CLEAR type. /// Dispatches to the correct cleanup function based on structural type analysis. /// For types that need no cleanup (primitives, enums, plain structs without RC fields), @@ -3991,12 +4055,12 @@ pub const CheatLib = struct { // 6. Set(U) if (comptime isSetType(T)) { - // Release owned keys before freeing the backing map. Interned - // sets never own their elements — backing map only. - const set_interned = comptime @hasDecl(T, "interned_elements") and T.interned_elements; + // Release owned keys before freeing the backing map. A set of + // Symbols needs no exemption: dupe/cleanup of a Symbol are a bit + // copy and a no-op, so the uniform path is already correct. const InnerMap = @TypeOf(ptr.inner); const inner_info = @typeInfo(InnerMap); - if (!set_interned and inner_info == .@"struct") { + if (inner_info == .@"struct") { var it = ptr.inner.keyIterator(); while (it.next()) |key_ptr| { const KeyT = @TypeOf(key_ptr.*); @@ -4158,6 +4222,13 @@ pub const CheatLib = struct { return if (value.len > 0) try alloc.dupe(u8, value) else value; } + // The mirror of the optional case below: a copy whose DESTINATION is a + // concrete T can be fed an already-narrowed `?T` source. The narrowing + // is the caller's proof of presence, so unwrap and copy the payload. + if (comptime info != .optional and @typeInfo(@TypeOf(value)) == .optional) { + return try dupeValue(T, value.?, alloc); + } + // Copy and drop are one compiler-generated semantic contract. A type // with drop glue but no clone glue is linear; reaching this path means // annotation/lowering failed to reject an illegal COPY. @@ -4348,10 +4419,9 @@ pub const CheatLib = struct { errdefer result.deinit(alloc); var src_mut = value; var it = src_mut.keyIterator(); - const set_interned = comptime @hasDecl(T, "interned_elements") and T.interned_elements; while (it.next()) |k| { const ElemT = @TypeOf(k.*); - const copied = if (comptime !set_interned and needsCleanup(ElemT)) + const copied = if (comptime needsCleanup(ElemT)) try dupeValue(ElemT, k.*, alloc) else k.*; diff --git a/zig/runtime/runtime.zig b/zig/runtime/runtime.zig index 1839f30d0..c13b680eb 100644 --- a/zig/runtime/runtime.zig +++ b/zig/runtime/runtime.zig @@ -389,6 +389,17 @@ pub const Runtime = struct { // Frame Allocator Backing + /// Safety builds validate what the frame allocator is asked to free. + /// Opt out with `pub const CLEAR_DISABLE_ARENA_FREE_CHECK = true;` in root. + const arena_free_check = blk: { + const mode = @import("builtin").mode; + if (mode != .Debug and mode != .ReleaseSafe) break :blk false; + if (@hasDecl(@import("root"), "CLEAR_DISABLE_ARENA_FREE_CHECK")) { + break :blk !@import("root").CLEAR_DISABLE_ARENA_FREE_CHECK; + } + break :blk true; + }; + pub const SmartAllocatorVTable = std.mem.Allocator.VTable{ .alloc = smartAlloc, .resize = smartResize, @@ -431,9 +442,25 @@ pub const Runtime = struct { fn smartFree(ctx: *anyopaque, buf: []u8, buf_align: std.mem.Alignment, ret_addr: usize) void { // We don't actually free individual items in a Frame/Arena model. // We just let them accumulate and wipe the slate clean at the end. - // But for correctness, we can forward the call if needed. - _ = ctx; - _ = buf; + // + // Because the free itself is a no-op, it accepts ANY pointer without + // complaint -- .rodata behind a string literal or a `String@symbol`, a + // heap value whose binding picked the wrong allocator, a borrow into a + // container someone else owns. Those are exactly the cleanup bugs the + // compiler can emit, and this path is where they go to hide: no leak, + // no crash, nothing for a test to observe. In safety builds, reject a + // pointer this arena never handed out. + const self = @as(*Runtime, @ptrCast(@alignCast(ctx))); + if (arena_free_check and buf.len > 0 and !self.overflow_arena.owns(buf.ptr)) { + std.debug.print( + "\n[CLEAR] frame free of memory this arena never allocated: ptr={x} len={d}\n" ++ + " A frame cleanup was emitted for a value the frame does not own.\n" ++ + " bytes: \"{s}\"\n", + .{ @intFromPtr(buf.ptr), buf.len, buf[0..@min(buf.len, 64)] }, + ); + std.debug.dumpCurrentStackTrace(.{}); + @panic("frame allocator asked to free foreign memory"); + } _ = buf_align; _ = ret_addr; } diff --git a/zig/runtime/symbol-intern-benchmark-test.zig b/zig/runtime/symbol-intern-benchmark-test.zig new file mode 100644 index 000000000..484aa67a2 --- /dev/null +++ b/zig/runtime/symbol-intern-benchmark-test.zig @@ -0,0 +1,178 @@ +// Benchmark: what does interning a symbol cost, and does the lock matter? +// +// `Runtime.internSymbol` takes `symbol_pool_lock` on every `.to_sym()`. The +// pool is a field on Runtime, and a fiber-local Runtime is touched by one +// thread at a time, so the question is whether that atomic can be skipped -- +// and whether skipping it is worth diverging from how Rust does this. +// +// The comparison isolates the SYNCHRONIZATION strategy: every variant uses the +// same hash map, the same allocator, and the same workload, so the delta is the +// locking discipline and nothing else. +// +// local_unlocked -- per-Runtime pool, no atomic (the proposal) +// local_locked -- per-Runtime pool, uncontended mutex (CLEAR today) +// global_1t -- one shared pool, one thread (ustr/rustc, best case) +// global_nt -- one shared pool, N threads (ustr/rustc, real case) +// +// Run it optimized -- in Debug the hash map dominates and the lock delta +// vanishes into noise: +// +// zig build benchmark -Doptimize=ReleaseFast +// +// Rust counterpart with the same workload and strategies is +// symbol-intern-benchmark.rs, for checking these numbers against a baseline. +// +// Measured (ReleaseFast, 8 cores), ns/op, Zig vs Rust: +// +// local_unlocked 12.9 / 20.4 local_locked (today) 18.6 / 25.8 +// global_1t 18.3 / 26.6 global_8t 138.7 / 228.3 +// +// Two conclusions. An uncontended mutex costs ~5.5 ns/op in BOTH languages, so +// that is a property of the atomic, not of Zig. And moving to ONE shared pool +// -- which a u32 index handle would require, since an index only means +// anything relative to its table -- costs 7.5x under 8-thread contention. +// That is the finding that matters: the per-Runtime pool is worth more than +// any handle-width saving. +// +// Workload is hit-dominated on purpose. In the self-hosted compiler 11,309 of +// 11,424 symbol uses are literals emitted as constants, and the 115 dynamic +// `symbol(expr)` sites re-intern names that almost always already exist. A +// miss-heavy benchmark would measure hash-map insertion, not interning. + +const std = @import("std"); +const compat = @import("../lib/compat.zig"); + +const HITS_PER_THREAD = 200_000; +const DISTINCT = 1352; // distinct symbols in the self-hosted compiler +const AVG_LEN = 17; // measured average symbol length, in bytes + +/// The pool as it exists today, minus the Runtime it hangs off. +const Pool = struct { + map: std.StringHashMapUnmanaged(void) = .empty, + lock: compat.Mutex = .{}, + alloc: std.mem.Allocator, + + fn deinit(self: *Pool) void { + var it = self.map.iterator(); + while (it.next()) |e| self.alloc.free(e.key_ptr.*); + self.map.deinit(self.alloc); + } + + fn internLocked(self: *Pool, value: []const u8) ![]const u8 { + self.lock.lock(); + defer self.lock.unlock(); + return self.internRaw(value); + } + + fn internUnlocked(self: *Pool, value: []const u8) ![]const u8 { + return self.internRaw(value); + } + + fn internRaw(self: *Pool, value: []const u8) ![]const u8 { + if (self.map.getKey(value)) |canonical| return canonical; + const canonical = try self.alloc.dupe(u8, value); + try self.map.put(self.alloc, canonical, {}); + return canonical; + } +}; + +fn makeNames(alloc: std.mem.Allocator) ![]const []const u8 { + const names = try alloc.alloc([]const u8, DISTINCT); + for (names, 0..) |*slot, i| { + var buf: [AVG_LEN]u8 = undefined; + for (&buf, 0..) |*c, j| c.* = 'a' + @as(u8, @intCast((i + j) % 26)); + // Keep them distinct: stamp the index over the tail. + _ = std.fmt.bufPrint(buf[AVG_LEN - 5 ..], "{d:0>5}", .{i}) catch unreachable; + slot.* = try alloc.dupe(u8, &buf); + } + return names; +} + +fn hammer(pool: *Pool, names: []const []const u8, locked: bool) void { + var i: usize = 0; + while (i < HITS_PER_THREAD) : (i += 1) { + const name = names[i % names.len]; + const got = if (locked) pool.internLocked(name) catch unreachable else pool.internUnlocked(name) catch unreachable; + std.mem.doNotOptimizeAway(got.ptr); + } +} + +test "Benchmark: symbol interning -- lock elision vs shared pool" { + const alloc = std.heap.c_allocator; + const names = try makeNames(alloc); + defer { + for (names) |n| alloc.free(n); + alloc.free(names); + } + + const thread_count: usize = @max(2, @min(8, std.Thread.getCpuCount() catch 4)); + + // 1. Per-Runtime pool, no lock. Only sound if a non-shared Runtime is + // provably touched by one thread at a time. + var unlocked_ns: u64 = 0; + { + var pool = Pool{ .alloc = alloc }; + defer pool.deinit(); + for (names) |n| _ = try pool.internUnlocked(n); // warm: measure hits + var timer = try compat.Timer.start(); + hammer(&pool, names, false); + unlocked_ns = timer.read(); + } + + // 2. Per-Runtime pool with today's mutex, uncontended. + var locked_ns: u64 = 0; + { + var pool = Pool{ .alloc = alloc }; + defer pool.deinit(); + for (names) |n| _ = try pool.internLocked(n); + var timer = try compat.Timer.start(); + hammer(&pool, names, true); + locked_ns = timer.read(); + } + + // 3. One shared pool, single thread: what ustr/rustc pay with no contention. + var global_1t_ns: u64 = 0; + { + var pool = Pool{ .alloc = alloc }; + defer pool.deinit(); + for (names) |n| _ = try pool.internLocked(n); + var timer = try compat.Timer.start(); + hammer(&pool, names, true); + global_1t_ns = timer.read(); + } + + // 4. One shared pool, N threads: what ustr/rustc pay in practice, and what + // CLEAR would adopt by moving to a global table for u32 indices. + var global_nt_ns: u64 = 0; + { + var pool = Pool{ .alloc = alloc }; + defer pool.deinit(); + for (names) |n| _ = try pool.internLocked(n); + + const threads = try alloc.alloc(std.Thread, thread_count); + defer alloc.free(threads); + + var timer = try compat.Timer.start(); + for (threads) |*t| t.* = try std.Thread.spawn(.{}, hammer, .{ &pool, names, true }); + for (threads) |t| t.join(); + global_nt_ns = timer.read(); + } + + const per = struct { + fn ns(total: u64, ops: u64) f64 { + return @as(f64, @floatFromInt(total)) / @as(f64, @floatFromInt(ops)); + } + }; + + const one: u64 = HITS_PER_THREAD; + const many: u64 = @as(u64, HITS_PER_THREAD) * @as(u64, thread_count); + + std.debug.print("\n=== symbol intern: {d} hits/thread over {d} distinct names ===\n", .{ HITS_PER_THREAD, DISTINCT }); + std.debug.print("local_unlocked (proposed) {d:>7.2} ns/op\n", .{per.ns(unlocked_ns, one)}); + std.debug.print("local_locked (today) {d:>7.2} ns/op\n", .{per.ns(locked_ns, one)}); + std.debug.print("global_1t (rust, 1T) {d:>7.2} ns/op\n", .{per.ns(global_1t_ns, one)}); + std.debug.print("global_{d}t (rust, {d}T) {d:>7.2} ns/op [{d} threads contending]\n", .{ thread_count, thread_count, per.ns(global_nt_ns, many), thread_count }); + std.debug.print("lock overhead (today vs proposed): {d:>5.2} ns/op\n", .{per.ns(locked_ns, one) - per.ns(unlocked_ns, one)}); + + try std.testing.expect(unlocked_ns > 0 and global_nt_ns > 0); +} diff --git a/zig/runtime/symbol-intern-benchmark.rs b/zig/runtime/symbol-intern-benchmark.rs new file mode 100644 index 000000000..430e28593 --- /dev/null +++ b/zig/runtime/symbol-intern-benchmark.rs @@ -0,0 +1,128 @@ +// Rust counterpart to zig/runtime/symbol-intern-benchmark-test.zig. +// +// rustc -O symbol-intern-benchmark.rs -o /tmp/symbench && /tmp/symbench +// +// Not part of any build; run it by hand when re-checking the numbers in the +// Zig benchmark's header against a Rust baseline. +// +// Same workload, same four synchronization strategies, so the Zig numbers can +// be read against a Rust baseline rather than against nothing. Uses std only +// (no crates.io), modelling the interner the way ustr does: canonical strings +// are leaked, so a handle is a &'static str and equality is pointer equality. + +use std::collections::HashSet; +use std::sync::Mutex; +use std::time::Instant; + +const HITS_PER_THREAD: usize = 200_000; +const DISTINCT: usize = 1352; +const AVG_LEN: usize = 17; + +fn make_names() -> Vec { + (0..DISTINCT) + .map(|i| { + let mut s: String = (0..AVG_LEN) + .map(|j| (b'a' + ((i + j) % 26) as u8) as char) + .collect(); + let tail = format!("{:05}", i); + s.truncate(AVG_LEN - 5); + s.push_str(&tail); + s + }) + .collect() +} + +fn intern_raw(set: &mut HashSet<&'static str>, value: &str) -> &'static str { + if let Some(found) = set.get(value) { + return found; + } + let leaked: &'static str = Box::leak(value.to_string().into_boxed_str()); + set.insert(leaked); + leaked +} + +fn main() { + let names = make_names(); + let threads: usize = std::thread::available_parallelism() + .map(|n| n.get().min(8).max(2)) + .unwrap_or(4); + + // 1. Local pool, no lock (the proposal). + let unlocked_ns = { + let mut set: HashSet<&'static str> = HashSet::new(); + for n in &names { + intern_raw(&mut set, n); + } + let t = Instant::now(); + for i in 0..HITS_PER_THREAD { + let got = intern_raw(&mut set, &names[i % names.len()]); + std::hint::black_box(got.as_ptr()); + } + t.elapsed().as_nanos() as f64 / HITS_PER_THREAD as f64 + }; + + // 2. Local pool behind an uncontended mutex (CLEAR today). + let locked_ns = { + let set: Mutex> = Mutex::new(HashSet::new()); + { + let mut g = set.lock().unwrap(); + for n in &names { + intern_raw(&mut g, n); + } + } + let t = Instant::now(); + for i in 0..HITS_PER_THREAD { + let mut g = set.lock().unwrap(); + let got = intern_raw(&mut g, &names[i % names.len()]); + std::hint::black_box(got.as_ptr()); + } + t.elapsed().as_nanos() as f64 / HITS_PER_THREAD as f64 + }; + + // 3+4. One shared pool: 1 thread, then N threads (ustr / rustc). + let global: &'static Mutex> = + Box::leak(Box::new(Mutex::new(HashSet::new()))); + { + let mut g = global.lock().unwrap(); + for n in &names { + intern_raw(&mut g, n); + } + } + + let global_1t_ns = { + let t = Instant::now(); + for i in 0..HITS_PER_THREAD { + let mut g = global.lock().unwrap(); + let got = intern_raw(&mut g, &names[i % names.len()]); + std::hint::black_box(got.as_ptr()); + } + t.elapsed().as_nanos() as f64 / HITS_PER_THREAD as f64 + }; + + let global_nt_ns = { + let names: &'static Vec = Box::leak(Box::new(names.clone())); + let t = Instant::now(); + let hs: Vec<_> = (0..threads) + .map(|_| { + std::thread::spawn(move || { + for i in 0..HITS_PER_THREAD { + let mut g = global.lock().unwrap(); + let got = intern_raw(&mut g, &names[i % names.len()]); + std::hint::black_box(got.as_ptr()); + } + }) + }) + .collect(); + for h in hs { + h.join().unwrap(); + } + t.elapsed().as_nanos() as f64 / (HITS_PER_THREAD * threads) as f64 + }; + + println!("=== rust: {} hits/thread over {} distinct names ===", HITS_PER_THREAD, DISTINCT); + println!("local_unlocked (proposed) {:>7.2} ns/op", unlocked_ns); + println!("local_locked (today) {:>7.2} ns/op", locked_ns); + println!("global_1t (rust, 1T) {:>7.2} ns/op", global_1t_ns); + println!("global_{}t (rust, {}T) {:>7.2} ns/op [{} threads contending]", threads, threads, global_nt_ns, threads); + println!("lock overhead (today vs proposed): {:>5.2} ns/op", locked_ns - unlocked_ns); +} diff --git a/zig/symbol-intern-benchmark-test.zig b/zig/symbol-intern-benchmark-test.zig new file mode 100644 index 000000000..a20bece71 --- /dev/null +++ b/zig/symbol-intern-benchmark-test.zig @@ -0,0 +1,3 @@ +test { + _ = @import("runtime/symbol-intern-benchmark-test.zig"); +}