From 99855ded7725d67531ad06327362dc881364c708 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Feb 2026 11:08:14 -0800 Subject: [PATCH 1/6] ZJIT: Lightweight Frames Co-authored-by: Alan Wu --- cont.c | 10 +- depend | 7 ++ error.c | 3 +- eval.c | 5 +- gc.c | 10 +- insns.def | 12 +-- jit.c | 5 + proc.c | 3 + thread.c | 5 +- vm.c | 96 ++++++++++++----- vm_backtrace.c | 53 +++++----- vm_core.h | 4 +- vm_dump.c | 31 +++--- vm_eval.c | 9 +- vm_insnhelper.c | 102 ++++++++++-------- yjit.c | 2 +- zjit.h | 68 ++++++++++++ zjit/bindgen/src/main.rs | 3 +- zjit/src/backend/arm64/mod.rs | 2 +- zjit/src/backend/lir.rs | 24 ++++- zjit/src/backend/x86_64/mod.rs | 2 +- zjit/src/codegen.rs | 98 ++++++++++++++---- zjit/src/cruby.rs | 9 +- zjit/src/cruby_bindings.inc.rs | 10 ++ zjit/src/gc.rs | 26 +++++ zjit/src/hir.rs | 2 +- zjit/src/jit_frame.rs | 184 +++++++++++++++++++++++++++++++++ zjit/src/lib.rs | 1 + zjit/src/payload.rs | 2 + zjit/src/state.rs | 23 ++++- 30 files changed, 635 insertions(+), 176 deletions(-) create mode 100644 zjit/src/jit_frame.rs diff --git a/cont.c b/cont.c index 2baf52a61a9999..1a5f6cb54fb680 100644 --- a/cont.c +++ b/cont.c @@ -41,6 +41,7 @@ extern int madvise(caddr_t, size_t, int); #include "vm_sync.h" #include "id_table.h" #include "ractor_core.h" +#include "zjit.h" static const int DEBUG = 0; @@ -1347,8 +1348,11 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) { - callback(cfp->iseq, data); + if ((cfp->pc || CFP_JIT_RETURN(cfp)) && (cfp->iseq || CFP_JIT_RETURN(cfp))) { + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) { + callback(iseq, data); + } } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } @@ -1368,7 +1372,7 @@ rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (cfp->jit_return && cfp->jit_return != leave_exception) { + if (CFP_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) { ((rb_control_frame_t *)cfp)->jit_return = leave_exit; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); diff --git a/depend b/depend index 8481ad8222873c..f391d2f887e75e 100644 --- a/depend +++ b/depend @@ -2620,6 +2620,7 @@ cont.$(OBJEXT): {$(VPATH)}vm_debug.h cont.$(OBJEXT): {$(VPATH)}vm_opts.h cont.$(OBJEXT): {$(VPATH)}vm_sync.h cont.$(OBJEXT): {$(VPATH)}yjit.h +cont.$(OBJEXT): {$(VPATH)}zjit.h debug.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h debug.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h debug.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -5420,6 +5421,7 @@ error.$(OBJEXT): {$(VPATH)}vm_opts.h error.$(OBJEXT): {$(VPATH)}vm_sync.h error.$(OBJEXT): {$(VPATH)}warning.rbinc error.$(OBJEXT): {$(VPATH)}yjit.h +error.$(OBJEXT): {$(VPATH)}zjit.h eval.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h eval.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h eval.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -5689,6 +5691,7 @@ eval.$(OBJEXT): {$(VPATH)}vm_core.h eval.$(OBJEXT): {$(VPATH)}vm_debug.h eval.$(OBJEXT): {$(VPATH)}vm_opts.h eval.$(OBJEXT): {$(VPATH)}vm_sync.h +eval.$(OBJEXT): {$(VPATH)}zjit.h explicit_bzero.$(OBJEXT): {$(VPATH)}config.h explicit_bzero.$(OBJEXT): {$(VPATH)}explicit_bzero.c explicit_bzero.$(OBJEXT): {$(VPATH)}internal/attr/format.h @@ -8072,6 +8075,7 @@ jit.$(OBJEXT): {$(VPATH)}vm_core.h jit.$(OBJEXT): {$(VPATH)}vm_debug.h jit.$(OBJEXT): {$(VPATH)}vm_opts.h jit.$(OBJEXT): {$(VPATH)}vm_sync.h +jit.$(OBJEXT): {$(VPATH)}zjit.h load.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h load.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h load.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -18615,6 +18619,7 @@ thread.$(OBJEXT): {$(VPATH)}vm_core.h thread.$(OBJEXT): {$(VPATH)}vm_debug.h thread.$(OBJEXT): {$(VPATH)}vm_opts.h thread.$(OBJEXT): {$(VPATH)}vm_sync.h +thread.$(OBJEXT): {$(VPATH)}zjit.h time.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h time.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h time.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -20210,6 +20215,7 @@ vm_backtrace.$(OBJEXT): {$(VPATH)}vm_core.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_debug.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_opts.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_sync.h +vm_backtrace.$(OBJEXT): {$(VPATH)}zjit.h vm_dump.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h vm_dump.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h vm_dump.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -20458,6 +20464,7 @@ vm_dump.$(OBJEXT): {$(VPATH)}vm_core.h vm_dump.$(OBJEXT): {$(VPATH)}vm_debug.h vm_dump.$(OBJEXT): {$(VPATH)}vm_dump.c vm_dump.$(OBJEXT): {$(VPATH)}vm_opts.h +vm_dump.$(OBJEXT): {$(VPATH)}zjit.h vm_sync.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h vm_sync.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h vm_sync.$(OBJEXT): $(CCAN_DIR)/list/list.h diff --git a/error.c b/error.c index 140049fdd2cc2b..f5606520cd5dab 100644 --- a/error.c +++ b/error.c @@ -50,6 +50,7 @@ #include "ruby_assert.h" #include "vm_core.h" #include "yjit.h" +#include "zjit.h" #include "builtin.h" @@ -2367,7 +2368,7 @@ name_err_init_attr(VALUE exc, VALUE recv, VALUE method) rb_ivar_set(exc, id_name, method); err_init_recv(exc, recv); if (cfp && VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) { - rb_ivar_set(exc, id_iseq, rb_iseqw_new(cfp->iseq)); + rb_ivar_set(exc, id_iseq, rb_iseqw_new(rb_zjit_cfp_iseq(cfp))); } return exc; } diff --git a/eval.c b/eval.c index 7d5ae75e3e6ee9..a30a36e4745a94 100644 --- a/eval.c +++ b/eval.c @@ -37,6 +37,7 @@ #include "ruby/vm.h" #include "vm_core.h" #include "ractor_core.h" +#include "zjit.h" NORETURN(static void rb_raise_jump(VALUE, VALUE)); void rb_ec_clear_current_thread_trace_func(const rb_execution_context_t *ec); @@ -2010,10 +2011,10 @@ errinfo_place(const rb_execution_context_t *ec) while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { if (VM_FRAME_RUBYFRAME_P(cfp)) { - if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE) { + if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } - else if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_ENSURE && + else if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_ENSURE && !THROW_DATA_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR]) && !FIXNUM_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR])) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; diff --git a/gc.c b/gc.c index 5001bc5c01cb2e..2f4dc39675b03a 100644 --- a/gc.c +++ b/gc.c @@ -1000,11 +1000,11 @@ gc_validate_pc(VALUE obj) rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *cfp = ec->cfp; - if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && cfp->pc) { - const VALUE *iseq_encoded = ISEQ_BODY(cfp->iseq)->iseq_encoded; - const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size; - RUBY_ASSERT(cfp->pc >= iseq_encoded, "PC not set when allocating, breaking tracing"); - RUBY_ASSERT(cfp->pc <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && (cfp->pc || CFP_JIT_RETURN(cfp))) { + const VALUE *iseq_encoded = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded; + const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_size; + RUBY_ASSERT(rb_zjit_cfp_pc(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); + RUBY_ASSERT(rb_zjit_cfp_pc(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); } #endif } diff --git a/insns.def b/insns.def index 92ae7c181628b9..df4147efdeacf8 100644 --- a/insns.def +++ b/insns.def @@ -926,7 +926,7 @@ opt_new // The bookkeeping slot should be empty. RUBY_ASSERT(TOPN(argc + 1) == Qnil); - if (vm_method_cfunc_is(GET_ISEQ(), cd, val, rb_class_new_instance_pass_kw)) { + if (vm_method_cfunc_is(GET_CFP(), cd, val, rb_class_new_instance_pass_kw)) { RB_DEBUG_COUNTER_INC(opt_new_hit); val = rb_obj_alloc(val); TOPN(argc) = val; @@ -947,7 +947,7 @@ objtostring // attr bool leaf = false; // attr bool zjit_profile = true; { - val = vm_objtostring(GET_ISEQ(), recv, cd); + val = vm_objtostring(GET_CFP(), recv, cd); if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); @@ -1006,7 +1006,7 @@ opt_nil_p (VALUE val) // attr bool zjit_profile = true; { - val = vm_opt_nil_p(GET_ISEQ(), cd, recv); + val = vm_opt_nil_p(GET_CFP(), cd, recv); if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); @@ -1435,7 +1435,7 @@ opt_eq (VALUE val) // attr bool zjit_profile = true; { - val = opt_equality(GET_ISEQ(), recv, obj, cd); + val = opt_equality(GET_CFP(), recv, obj, cd); if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); @@ -1450,7 +1450,7 @@ opt_neq (VALUE val) // attr bool zjit_profile = true; { - val = vm_opt_neq(GET_ISEQ(), cd, cd_eq, recv, obj); + val = vm_opt_neq(GET_CFP(), cd, cd_eq, recv, obj); if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); @@ -1672,7 +1672,7 @@ opt_not (VALUE val) // attr bool zjit_profile = true; { - val = vm_opt_not(GET_ISEQ(), cd, recv); + val = vm_opt_not(GET_CFP(), cd, recv); if (UNDEF_P(val)) { CALL_SIMPLE_METHOD(); diff --git a/jit.c b/jit.c index 11ff015e362c3d..a87becf0a9259e 100644 --- a/jit.c +++ b/jit.c @@ -19,6 +19,7 @@ #include "internal/class.h" #include "internal/imemo.h" #include "ruby/internal/core/rtypeddata.h" +#include "zjit.h" #ifndef _WIN32 #include @@ -522,6 +523,10 @@ void rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc) { cfp->pc = pc; + if (rb_zjit_enabled_p/* && cfp->jit_return*/) { + //cfp->iseq = rb_zjit_jit_return_iseq(cfp->jit_return); + cfp->jit_return = 0; // TODO: do it in Rust (function_stub_hit) + } } void diff --git a/proc.c b/proc.c index 3e2afeab3ca1bb..32086a759dd47d 100644 --- a/proc.c +++ b/proc.c @@ -1285,6 +1285,9 @@ rb_proc_arity(VALUE self) static void block_setup(struct rb_block *block, VALUE block_handler) { + // TODO: do we need to do something about this for ZJIT? + // we copy captured block but not jit_return. + // do we need to query block_code for specialized C method calls? switch (vm_block_handler_type(block_handler)) { case block_handler_type_iseq: block->type = block_type_iseq; diff --git a/thread.c b/thread.c index 99252168defe3a..8c815494d7e24d 100644 --- a/thread.c +++ b/thread.c @@ -99,6 +99,7 @@ #include "ractor_core.h" #include "vm_debug.h" #include "vm_sync.h" +#include "zjit.h" #include "ccan/list/list.h" @@ -5915,7 +5916,7 @@ update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) VALUE num; void rb_iseq_clear_event_flags(const rb_iseq_t *iseq, size_t pos, rb_event_flag_t reset); if (GET_VM()->coverage_mode & COVERAGE_TARGET_ONESHOT_LINES) { - rb_iseq_clear_event_flags(cfp->iseq, cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); + rb_iseq_clear_event_flags(cfp->iseq, rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); rb_ary_push(lines, LONG2FIX(line + 1)); return; } @@ -5940,7 +5941,7 @@ update_branch_coverage(VALUE data, const rb_trace_arg_t *trace_arg) if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE branches = RARRAY_AREF(coverage, COVERAGE_INDEX_BRANCHES); if (branches) { - long pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1; + long pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1; long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(cfp->iseq), pc)), count; VALUE counters = RARRAY_AREF(branches, 1); VALUE num = RARRAY_AREF(counters, idx); diff --git a/vm.c b/vm.c index 916e379d671a1e..323ce0c91584f5 100644 --- a/vm.c +++ b/vm.c @@ -931,7 +931,7 @@ rb_control_frame_t * rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - if (cfp->iseq) { + if (cfp->iseq || CFP_JIT_RETURN(cfp)) { return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1109,13 +1109,14 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } } + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); if (!VM_FRAME_RUBYFRAME_P(cfp)) { local_size = VM_ENV_DATA_SIZE; } else { - local_size = ISEQ_BODY(cfp->iseq)->local_table_size; - if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { - int ci_offset = local_size - ISEQ_BODY(cfp->iseq)->param.size + VM_ENV_DATA_SIZE; + local_size = ISEQ_BODY(iseq)->local_table_size; + if (ISEQ_BODY(iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int ci_offset = local_size - ISEQ_BODY(iseq)->param.size + VM_ENV_DATA_SIZE; CALL_INFO ci = (CALL_INFO)VM_CF_LEP(cfp)[-ci_offset]; local_size += vm_ci_argc(ci); @@ -1127,8 +1128,8 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co // This is done before creating the imemo_env because VM_STACK_ENV_WRITE // below leaves the on-stack ep in a state that is unsafe to GC. if (VM_FRAME_RUBYFRAME_P(cfp)) { - rb_yjit_invalidate_ep_is_bp(cfp->iseq); - rb_zjit_invalidate_no_ep_escape(cfp->iseq); + rb_yjit_invalidate_ep_is_bp(iseq); + rb_zjit_invalidate_no_ep_escape(iseq); } /* @@ -1156,7 +1157,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co env_ep = &env_body[local_size - 1 /* specval */]; env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env; - env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL); + env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? iseq : NULL); env->ep = env_ep; env->env = env_body; env->env_size = env_size; @@ -1678,8 +1679,8 @@ rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *s GetBindingPtr(bindval, bind); vm_bind_update_env(bindval, bind, envval); RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self); - RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, cfp->iseq); - RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(ruby_level_cfp->iseq)->location.pathobj); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, rb_zjit_cfp_iseq(cfp)); + RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(rb_zjit_cfp_iseq(ruby_level_cfp))->location.pathobj); bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp); return bindval; @@ -1986,9 +1987,9 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s VALUE * rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (cfp->pc == 0 || cfp->iseq == 0) { + while ((cfp->pc == 0 && !CFP_JIT_RETURN(cfp)) || (cfp->iseq == 0 && !CFP_JIT_RETURN(cfp))) { if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { - struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq; + struct vm_ifunc *ifunc = (struct vm_ifunc *)rb_zjit_cfp_iseq(cfp); return ifunc->svar_lep; } else { @@ -2071,7 +2072,7 @@ rb_sourcefile(void) const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return RSTRING_PTR(rb_iseq_path(cfp->iseq)); + return RSTRING_PTR(rb_iseq_path(rb_zjit_cfp_iseq(cfp))); } else { return 0; @@ -2100,7 +2101,7 @@ rb_source_location(int *pline) if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) { if (pline) *pline = rb_vm_get_sourceline(cfp); - return rb_iseq_path(cfp->iseq); + return rb_iseq_path(rb_zjit_cfp_iseq(cfp)); } else { if (pline) *pline = 0; @@ -2582,7 +2583,7 @@ hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, in return; } else { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(ec->cfp); rb_hook_list_t *local_hooks = NULL; unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; if (RB_UNLIKELY(local_hooks_cnt > 0)) { @@ -2835,6 +2836,31 @@ vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, return result; } +static inline void +zjit_materialize_frames(rb_control_frame_t *cfp) +{ + if (!rb_zjit_enabled_p) return; + + while (true) { + if (CFP_JIT_RETURN(cfp)) { + cfp->pc = rb_zjit_cfp_pc(cfp); + cfp->iseq = rb_zjit_cfp_iseq(cfp); + if (rb_zjit_jit_return_materialize_block_code(cfp->jit_return)) { + cfp->block_code = NULL; + } + cfp->jit_return = 0; + } + if (VM_FRAME_FINISHED_P(cfp)) break; + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } +} + +void +rb_zjit_materialize_frames(rb_control_frame_t *cfp) +{ + zjit_materialize_frames(cfp); +} + static inline VALUE vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo) { @@ -2852,7 +2878,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V cont_pc = cont_sp = 0; catch_iseq = NULL; - while (ec->cfp->pc == 0 || ec->cfp->iseq == 0) { + while (rb_zjit_cfp_pc(ec->cfp) == 0 || rb_zjit_cfp_iseq(ec->cfp) == 0) { if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) { EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self, rb_vm_frame_method_entry(ec->cfp)->def->original_id, @@ -2866,7 +2892,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } rb_control_frame_t *const cfp = ec->cfp; - epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + epc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded; escape_cfp = NULL; if (state == TAG_BREAK || state == TAG_RETURN) { @@ -2879,7 +2905,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V THROW_DATA_STATE_SET(err, state = TAG_BREAK); } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2906,13 +2932,14 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* TAG_BREAK */ *cfp->sp++ = THROW_DATA_VAL(err); ec->errinfo = Qnil; + zjit_materialize_frames(cfp); return Qundef; } } } if (state == TAG_RAISE) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2928,7 +2955,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else if (state == TAG_RETRY) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2943,7 +2970,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const rb_control_frame_t *escape_cfp; escape_cfp = THROW_DATA_CATCH_FRAME(err); if (cfp == escape_cfp) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + zjit_materialize_frames(cfp); + cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + entry->cont; ec->errinfo = Qnil; return Qundef; } @@ -2961,7 +2989,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* otherwise = dontcare */ }[state]; - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); @@ -2973,7 +3001,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V break; } else if (entry->type == type) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + zjit_materialize_frames(cfp); + cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + entry->cont; cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { @@ -2987,7 +3016,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -3007,8 +3036,9 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const int arg_size = 1; rb_iseq_check(catch_iseq); + zjit_materialize_frames(cfp); // vm_base_ptr looks at cfp->iseq cfp->sp = vm_base_ptr(cfp) + cont_sp; - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + cont_pc; + cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + cont_pc; /* push block frame */ cfp->sp[0] = (VALUE)err; @@ -3630,8 +3660,17 @@ rb_execution_context_update(rb_execution_context_t *ec) while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; cfp->self = rb_gc_location(cfp->self); - cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); - cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + rb_zjit_jit_return_set_iseq(cfp->jit_return, (rb_iseq_t *)rb_gc_location((VALUE)rb_zjit_cfp_iseq(cfp))); + // block_code may have been written by the JIT caller (gen_block_handler_specval) + // for passing blocks. It must be relocated even when jit_return is set, because + // Proc creation (block_setup) copies it from the CFP's captured block. + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } + else { + cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } if (!VM_ENV_LOCAL_P(ep)) { const VALUE *prev_ep = VM_ENV_PREV_EP(ep); @@ -3682,8 +3721,9 @@ rb_execution_context_mark(const rb_execution_context_t *ec) VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep)); rb_gc_mark_movable(cfp->self); - rb_gc_mark_movable((VALUE)cfp->iseq); - rb_gc_mark_movable((VALUE)cfp->block_code); + // TODO: use a broader `if` for rb_zjit_cfp_iseq instead of calling rb_zjit_cfp_* every time + rb_gc_mark_movable((VALUE)rb_zjit_cfp_iseq(cfp)); + rb_gc_mark_movable((VALUE)rb_zjit_cfp_block_code(cfp)); if (VM_ENV_LOCAL_P(ep) && VM_ENV_BOXED_P(ep)) { const rb_box_t *box = VM_ENV_BOX(ep); diff --git a/vm_backtrace.c b/vm_backtrace.c index 07d2e33e321787..715443deb824ec 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -18,6 +18,7 @@ #include "ruby/debug.h" #include "ruby/encoding.h" #include "vm_core.h" +#include "zjit.h" static VALUE rb_cBacktrace; static VALUE rb_cBacktraceLocation; @@ -101,9 +102,9 @@ calc_node_id(const rb_iseq_t *iseq, const VALUE *pc) int rb_vm_get_sourceline(const rb_control_frame_t *cfp) { - if (VM_FRAME_RUBYFRAME_P(cfp) && cfp->iseq) { - const rb_iseq_t *iseq = cfp->iseq; - int line = calc_lineno(iseq, cfp->pc); + if (VM_FRAME_RUBYFRAME_P(cfp) && (cfp->iseq || CFP_JIT_RETURN(cfp))) { + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + int line = calc_lineno(iseq, rb_zjit_cfp_pc(cfp)); if (line != 0) { return line; } @@ -617,7 +618,7 @@ backtrace_size(const rb_execution_context_t *ec) static bool is_rescue_or_ensure_frame(const rb_control_frame_t *cfp) { - enum rb_iseq_type type = ISEQ_BODY(cfp->iseq)->type; + enum rb_iseq_type type = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type; return type == ISEQ_TYPE_RESCUE || type == ISEQ_TYPE_ENSURE; } @@ -687,17 +688,17 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram } for (; cfp != end_cfp && (bt->backtrace_size < num_frames); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (cfp->iseq) { - if (cfp->pc) { + if (cfp->iseq || CFP_JIT_RETURN(cfp)) { + if (cfp->pc || CFP_JIT_RETURN(cfp)) { if (start_frame > 0) { start_frame--; } else { - bool internal = is_internal_location(cfp->iseq); + bool internal = is_internal_location(rb_zjit_cfp_iseq(cfp)); if (skip_internal && internal) continue; if (!skip_next_frame) { - const rb_iseq_t *iseq = cfp->iseq; - const VALUE *pc = cfp->pc; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const VALUE *pc = rb_zjit_cfp_pc(cfp); if (internal && backpatch_counter > 0) { // To keep only one internal frame, discard the previous backpatch frames bt->backtrace_size -= backpatch_counter; @@ -752,10 +753,10 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram // is the one of the caller Ruby frame, so if the last entry is a C frame we find the caller Ruby frame here. if (backpatch_counter > 0) { for (; cfp != end_cfp; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (cfp->iseq && cfp->pc && !(skip_internal && is_internal_location(cfp->iseq))) { + if ((cfp->iseq || CFP_JIT_RETURN(cfp)) && (cfp->pc || CFP_JIT_RETURN(cfp)) && !(skip_internal && is_internal_location(rb_zjit_cfp_iseq(cfp)))) { VM_ASSERT(!skip_next_frame); // ISEQ_TYPE_RESCUE/ISEQ_TYPE_ENSURE should have a caller Ruby ISEQ, not a cfunc - bt_backpatch_loc(backpatch_counter, loc, cfp->iseq, cfp->pc); - RB_OBJ_WRITTEN(btobj, Qundef, cfp->iseq); + bt_backpatch_loc(backpatch_counter, loc, rb_zjit_cfp_iseq(cfp), rb_zjit_cfp_pc(cfp)); + RB_OBJ_WRITTEN(btobj, Qundef, rb_zjit_cfp_iseq(cfp)); if (do_yield) { bt_yield_loc(loc - backpatch_counter, backpatch_counter, btobj); } @@ -1019,8 +1020,8 @@ backtrace_each(const rb_execution_context_t *ec, /* SDR(); */ for (i=0, cfp = start_cfp; ivm_stack + ec->vm_stack_size) - cfp); */ - if (cfp->iseq) { - if (cfp->pc) { + if (cfp->iseq || CFP_JIT_RETURN(cfp)) { + if (cfp->pc || CFP_JIT_RETURN(cfp)) { iter_iseq(arg, cfp); } } @@ -1052,8 +1053,8 @@ oldbt_init(void *ptr, size_t dmy) static void oldbt_iter_iseq(void *ptr, const rb_control_frame_t *cfp) { - const rb_iseq_t *iseq = cfp->iseq; - const VALUE *pc = cfp->pc; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const VALUE *pc = rb_zjit_cfp_pc(cfp); struct oldbt_arg *arg = (struct oldbt_arg *)ptr; VALUE file = arg->filename = rb_iseq_path(iseq); VALUE name = ISEQ_BODY(iseq)->location.label; @@ -1550,17 +1551,18 @@ collect_caller_bindings_iseq(void *arg, const rb_control_frame_t *cfp) { struct collect_caller_bindings_data *data = (struct collect_caller_bindings_data *)arg; VALUE frame = rb_ary_new2(6); + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); rb_ary_store(frame, CALLER_BINDING_SELF, cfp->self); rb_ary_store(frame, CALLER_BINDING_CLASS, get_klass(cfp)); rb_ary_store(frame, CALLER_BINDING_BINDING, GC_GUARDED_PTR(cfp)); /* create later */ - rb_ary_store(frame, CALLER_BINDING_ISEQ, cfp->iseq ? (VALUE)cfp->iseq : Qnil); + rb_ary_store(frame, CALLER_BINDING_ISEQ, iseq ? (VALUE)iseq : Qnil); rb_ary_store(frame, CALLER_BINDING_CFP, GC_GUARDED_PTR(cfp)); rb_backtrace_location_t *loc = &data->bt->backtrace[data->bt->backtrace_size++]; RB_OBJ_WRITE(data->btobj, &loc->cme, rb_vm_frame_method_entry(cfp)); - RB_OBJ_WRITE(data->btobj, &loc->iseq, cfp->iseq); - loc->pc = cfp->pc; + RB_OBJ_WRITE(data->btobj, &loc->iseq, iseq); + loc->pc = rb_zjit_cfp_pc(cfp); VALUE vloc = location_create(loc, (void *)data->btobj); rb_ary_store(frame, CALLER_BINDING_LOC, vloc); @@ -1745,7 +1747,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b end_cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp); for (i=0; ipc != 0) { + if (VM_FRAME_RUBYFRAME_P_UNCHECKED(cfp) && (cfp->pc != 0 || CFP_JIT_RETURN(cfp))) { if (start > 0) { start--; continue; @@ -1753,17 +1755,18 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b /* record frame info */ cme = rb_vm_frame_method_entry_unchecked(cfp); + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); if (cme && cme->def->type == VM_METHOD_TYPE_ISEQ) { buff[i] = (VALUE)cme; } else { - buff[i] = (VALUE)cfp->iseq; + buff[i] = (VALUE)iseq; } if (lines) { - const VALUE *pc = cfp->pc; - VALUE *iseq_encoded = ISEQ_BODY(cfp->iseq)->iseq_encoded; - VALUE *pc_end = iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size; + const VALUE *pc = rb_zjit_cfp_pc(cfp); + VALUE *iseq_encoded = ISEQ_BODY(iseq)->iseq_encoded; + VALUE *pc_end = iseq_encoded + ISEQ_BODY(iseq)->iseq_size; // The topmost frame may have an invalid PC because the JIT // may leave it uninitialized for speed. JIT code must update the PC @@ -1778,7 +1781,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b lines[i] = 0; } else { - lines[i] = calc_lineno(cfp->iseq, pc); + lines[i] = calc_lineno(iseq, pc); } } diff --git a/vm_core.h b/vm_core.h index 85711ffc7e8113..87d2661c0f9d4f 100644 --- a/vm_core.h +++ b/vm_core.h @@ -1532,8 +1532,8 @@ static inline int VM_FRAME_CFRAME_P(const rb_control_frame_t *cfp) { int cframe_p = VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_CFRAME) != 0; - VM_ASSERT(RUBY_VM_NORMAL_ISEQ_P(cfp->iseq) != cframe_p || - (VM_FRAME_TYPE(cfp) & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY); + //VM_ASSERT(RUBY_VM_NORMAL_ISEQ_P(cfp->iseq) != cframe_p || + // (VM_FRAME_TYPE(cfp) & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY); return cframe_p; } diff --git a/vm_dump.c b/vm_dump.c index e2b4804ab0d583..a0398a66ebcc3d 100644 --- a/vm_dump.c +++ b/vm_dump.c @@ -37,6 +37,7 @@ #include "iseq.h" #include "vm_core.h" #include "ractor_core.h" +#include "zjit.h" #define MAX_POSBUF 128 @@ -118,21 +119,21 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c selfstr = ""; } - if (cfp->iseq != 0) { + if (cfp->iseq || CFP_JIT_RETURN(cfp)) { + iseq = rb_zjit_cfp_iseq(cfp); #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - if (RUBY_VM_IFUNC_P(cfp->iseq)) { + if (RUBY_VM_IFUNC_P(iseq)) { iseq_name = ""; } - else if (SYMBOL_P((VALUE)cfp->iseq)) { - tmp = rb_sym2str((VALUE)cfp->iseq); + else if (SYMBOL_P((VALUE)iseq)) { + tmp = rb_sym2str((VALUE)iseq); iseq_name = RSTRING_PTR(tmp); snprintf(posbuf, MAX_POSBUF, ":%s", iseq_name); line = -1; } else { - if (cfp->pc) { - iseq = cfp->iseq; - pc = cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + if (cfp->pc || CFP_JIT_RETURN(cfp)) { + pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -169,6 +170,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c else { kprintf("b:---- "); } + kprintf("r:%p ", cfp->jit_return); kprintf("%-6s", magic); if (line) { kprintf(" %s", posbuf); @@ -339,21 +341,22 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro break; } - if (cfp && cfp->iseq != 0) { + if (cfp && (cfp->iseq != 0 || CFP_JIT_RETURN(cfp))) { #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - if (RUBY_VM_IFUNC_P(cfp->iseq)) { + const rb_iseq_t *resolved_iseq = rb_zjit_cfp_iseq(cfp); + if (RUBY_VM_IFUNC_P(resolved_iseq)) { iseq_name = ""; } - else if (SYMBOL_P((VALUE)cfp->iseq)) { - tmp = rb_sym2str((VALUE)cfp->iseq); + else if (SYMBOL_P((VALUE)resolved_iseq)) { + tmp = rb_sym2str((VALUE)resolved_iseq); iseq_name = RSTRING_PTR(tmp); snprintf(posbuf, MAX_POSBUF, ":%s", iseq_name); line = -1; } else { - if (cfp->pc) { - iseq = cfp->iseq; - pc = cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + if (cfp->pc || CFP_JIT_RETURN(cfp)) { + iseq = resolved_iseq; + pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); diff --git a/vm_eval.c b/vm_eval.c index 25d366f5cd19e6..02fd83ef699923 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1988,12 +1988,12 @@ eval_string_with_cref(VALUE self, VALUE src, rb_cref_t *cref, VALUE file, int li block.as.captured = *VM_CFP_TO_CAPTURED_BLOCK(cfp); block.as.captured.self = self; - block.as.captured.code.iseq = cfp->iseq; + block.as.captured.code.iseq = rb_zjit_cfp_iseq(cfp); block.type = block_type_iseq; // EP is not escaped to the heap here, but captured and reused by another frame. // ZJIT's locals are incompatible with it unlike YJIT's, so invalidate the ISEQ for ZJIT. - rb_zjit_invalidate_no_ep_escape(cfp->iseq); + rb_zjit_invalidate_no_ep_escape(rb_zjit_cfp_iseq(cfp)); iseq = eval_make_iseq(src, file, line, &block); if (!iseq) { @@ -2849,10 +2849,11 @@ rb_current_realfilepath(void) rb_control_frame_t *cfp = ec->cfp; cfp = vm_get_ruby_level_caller_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)); if (cfp != NULL) { - VALUE path = rb_iseq_realpath(cfp->iseq); + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + VALUE path = rb_iseq_realpath(iseq); if (RTEST(path)) return path; // eval context - path = rb_iseq_path(cfp->iseq); + path = rb_iseq_path(iseq); if (path == eval_default_path) { return Qnil; } diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 4e4ec36eb6729b..34f95d3feb13a3 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1767,16 +1767,16 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c escape_cfp = reg_cfp; while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) { - if (ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) { + if (ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp); ep = escape_cfp->ep; - base_iseq = escape_cfp->iseq; + base_iseq = rb_zjit_cfp_iseq(escape_cfp); } else { ep = VM_ENV_PREV_EP(ep); base_iseq = ISEQ_BODY(base_iseq)->parent_iseq; escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep); - VM_ASSERT(escape_cfp->iseq == base_iseq); + VM_ASSERT(rb_zjit_cfp_iseq(escape_cfp) == base_iseq); } } @@ -1790,8 +1790,8 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c while (escape_cfp < eocfp) { if (escape_cfp->ep == ep) { - const rb_iseq_t *const iseq = escape_cfp->iseq; - const VALUE epc = escape_cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + const rb_iseq_t *const iseq = rb_zjit_cfp_iseq(escape_cfp); + const VALUE epc = rb_zjit_cfp_pc(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded; const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table; unsigned int i; @@ -1850,7 +1850,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c if (lep == target_lep && VM_FRAME_RUBYFRAME_P(escape_cfp) && - ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) { + ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { in_class_frame = 1; target_lep = 0; } @@ -1880,7 +1880,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) { - switch (ISEQ_BODY(escape_cfp->iseq)->type) { + switch (ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type) { case ISEQ_TYPE_TOP: case ISEQ_TYPE_MAIN: if (toplevel) { @@ -1894,7 +1894,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } break; case ISEQ_TYPE_EVAL: { - const rb_iseq_t *is = escape_cfp->iseq; + const rb_iseq_t *is = rb_zjit_cfp_iseq(escape_cfp); enum rb_iseq_type t = ISEQ_BODY(is)->type; while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) { if (!(is = ISEQ_BODY(is)->parent_iseq)) break; @@ -1912,7 +1912,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } - if (escape_cfp->ep == target_lep && ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_METHOD) { + if (escape_cfp->ep == target_lep && ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_METHOD) { if (target_ep == NULL) { goto valid_return; } @@ -2352,9 +2352,9 @@ vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass) return cc; } -ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)); +ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass)); static const struct rb_callcache * -vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass) +vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE klass) { const struct rb_callcache *cc = cd->cc; @@ -2376,24 +2376,29 @@ vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass) } #endif - return vm_search_method_slowpath0(cd_owner, cd, klass); + return vm_search_method_slowpath0((VALUE)rb_zjit_cfp_iseq(reg_cfp), cd, klass); } static const struct rb_callable_method_entry_struct * -vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv) +vm_search_method(struct rb_control_frame_struct *reg_cfp, struct rb_call_data *cd, VALUE recv) { VALUE klass = CLASS_OF(recv); VM_ASSERT(klass != Qfalse); VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass)); - const struct rb_callcache *cc = vm_search_method_fastpath(cd_owner, cd, klass); + const struct rb_callcache *cc = vm_search_method_fastpath(reg_cfp, cd, klass); return vm_cc_cme(cc); } const struct rb_callable_method_entry_struct * rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv) { - return vm_search_method(cd_owner, cd, recv); + // TODO: change the code structure to use the inline cache again + // we need to query iseq differently during ZJIT compilation + // (ZJIT doesn't necessarily compile the iseq running on CFP) + VALUE klass = CLASS_OF(recv); + const struct rb_callcache *cc = vm_search_method_slowpath0(cd_owner, cd, klass); + return vm_cc_cme(cc); } #if __has_attribute(transparent_union) @@ -2453,10 +2458,10 @@ check_method_basic_definition(const rb_callable_method_entry_t *me) } static inline int -vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func) +vm_method_cfunc_is(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv, cfunc_type func) { - VM_ASSERT(iseq != NULL); - const struct rb_callable_method_entry_struct *cme = vm_search_method((VALUE)iseq, cd, recv); + VM_ASSERT(reg_cfp != NULL); + const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv); return check_cfunc(cme, func); } @@ -2469,11 +2474,17 @@ rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func int rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func) { - return vm_method_cfunc_is(iseq, cd, recv, func); + // TODO: change the code structure to use the inline cache again + // we need to query iseq differently during ZJIT compilation + // (ZJIT doesn't necessarily compile the iseq running on CFP) + VALUE klass = CLASS_OF(recv); + const struct rb_callcache *cc = vm_search_method_slowpath0((VALUE)iseq, cd, klass); + const struct rb_callable_method_entry_struct *cme = vm_cc_cme(cc); + return check_cfunc(cme, func); } #define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func)) -#define vm_method_cfunc_is(iseq, cd, recv, func) vm_method_cfunc_is(iseq, cd, recv, make_cfunc_type(func)) +#define vm_method_cfunc_is(reg_cfp, cd, recv, func) vm_method_cfunc_is(reg_cfp, cd, recv, make_cfunc_type(func)) #define EQ_UNREDEFINED_P(t) BASIC_OP_UNREDEFINED_P(BOP_EQ, t##_REDEFINED_OP_FLAG) @@ -2542,14 +2553,14 @@ opt_equality_specialized(VALUE recv, VALUE obj) } static VALUE -opt_equality(const rb_iseq_t *cd_owner, VALUE recv, VALUE obj, CALL_DATA cd) +opt_equality(struct rb_control_frame_struct *reg_cfp, VALUE recv, VALUE obj, CALL_DATA cd) { - VM_ASSERT(cd_owner != NULL); + VM_ASSERT(reg_cfp != NULL); VALUE val = opt_equality_specialized(recv, obj); if (!UNDEF_P(val)) return val; - if (!vm_method_cfunc_is(cd_owner, cd, recv, rb_obj_equal)) { + if (!vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_equal)) { return Qundef; } else { @@ -4696,7 +4707,7 @@ vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_c if (ref_cme) { if (calling->cd->cc) { const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement); - RB_OBJ_WRITE(cfp->iseq, &calling->cd->cc, cc); + RB_OBJ_WRITE(rb_zjit_cfp_iseq(cfp), &calling->cd->cc, cc); return vm_call_method(ec, cfp, calling); } else { @@ -5117,6 +5128,7 @@ static const struct rb_callcache * vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv) { VALUE current_defined_class; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(reg_cfp); const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp); if (!me) { @@ -5126,7 +5138,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c current_defined_class = vm_defined_class_for_protected_call(me); if (BUILTIN_TYPE(current_defined_class) != T_MODULE && - reg_cfp->iseq != method_entry_iseqptr(me) && + iseq != method_entry_iseqptr(me) && !rb_obj_is_kind_of(recv, current_defined_class)) { VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ? RCLASS_INCLUDER(current_defined_class) : current_defined_class; @@ -5158,7 +5170,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c vm_ci_argc(cd->ci), vm_ci_kwarg(cd->ci)); - RB_OBJ_WRITTEN(reg_cfp->iseq, Qundef, cd->ci); + RB_OBJ_WRITTEN(iseq, Qundef, cd->ci); } const struct rb_callcache *cc; @@ -5168,10 +5180,10 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c if (!klass) { /* bound instance method of module */ cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super); - RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc); + RB_OBJ_WRITE(iseq, &cd->cc, cc); } else { - cc = vm_search_method_fastpath((VALUE)reg_cfp->iseq, cd, klass); + cc = vm_search_method_fastpath(reg_cfp, cd, klass); const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc); // define_method can cache for different method id @@ -5183,7 +5195,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid); if (cme) { cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super); - RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc); + RB_OBJ_WRITE(iseq, &cd->cc, cc); } else { cd->cc = cc = empty_cc_for_super(); @@ -6123,7 +6135,7 @@ vm_sendish( switch (method_explorer) { case mexp_search_method: - calling.cc = cc = vm_search_method_fastpath((VALUE)reg_cfp->iseq, cd, CLASS_OF(recv)); + calling.cc = cc = vm_search_method_fastpath(reg_cfp, cd, CLASS_OF(recv)); val = vm_cc_call(cc)(ec, GET_CFP(), &calling); break; case mexp_search_super: @@ -6147,6 +6159,7 @@ rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd return val; } +// Fallback for YJIT/ZJIT, not used by the interpreter VALUE rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq) { @@ -6160,7 +6173,7 @@ rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(rb_zjit_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6189,6 +6202,7 @@ rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ return val; } +// Fallback for YJIT/ZJIT, not used by the interpreter VALUE rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq) { @@ -6201,7 +6215,7 @@ rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(rb_zjit_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6230,14 +6244,14 @@ VALUE rb_mod_to_s(VALUE); VALUE rb_mod_name(VALUE); static VALUE -vm_objtostring(const rb_iseq_t *iseq, VALUE recv, CALL_DATA cd) +vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd) { int type = TYPE(recv); if (type == T_STRING) { return recv; } - const struct rb_callable_method_entry_struct *cme = vm_search_method((VALUE)iseq, cd, recv); + const struct rb_callable_method_entry_struct *cme = vm_search_method(reg_cfp, cd, recv); switch (type) { case T_SYMBOL: @@ -6288,9 +6302,9 @@ vm_objtostring(const rb_iseq_t *iseq, VALUE recv, CALL_DATA cd) // ZJIT implementation is using the C function // and needs to call a non-static function VALUE -rb_vm_objtostring(const rb_iseq_t *iseq, VALUE recv, CALL_DATA cd) +rb_vm_objtostring(struct rb_control_frame_struct *reg_cfp, VALUE recv, CALL_DATA cd) { - return vm_objtostring(iseq, recv, cd); + return vm_objtostring(reg_cfp, recv, cd); } static VALUE @@ -6609,7 +6623,7 @@ rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const vm_ic_track_const_chain(GET_CFP(), ic, segments); // Undo the PC increment to get the address to this instruction // INSN_ATTR(width) == 2 - vm_ic_update(GET_ISEQ(), ic, val, GET_EP(), GET_PC() - 2); + vm_ic_update(rb_zjit_cfp_iseq(GET_CFP()), ic, val, GET_EP(), rb_zjit_cfp_pc(GET_CFP()) - 2); } return val; } @@ -6841,10 +6855,10 @@ vm_opt_mod(VALUE recv, VALUE obj) } static VALUE -vm_opt_neq(const rb_iseq_t *iseq, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj) +vm_opt_neq(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj) { - if (vm_method_cfunc_is(iseq, cd, recv, rb_obj_not_equal)) { - VALUE val = opt_equality(iseq, recv, obj, cd_eq); + if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not_equal)) { + VALUE val = opt_equality(reg_cfp, recv, obj, cd_eq); if (!UNDEF_P(val)) { return RBOOL(!RTEST(val)); @@ -7096,13 +7110,13 @@ vm_opt_empty_p(VALUE recv) VALUE rb_false(VALUE obj); static VALUE -vm_opt_nil_p(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv) +vm_opt_nil_p(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv) { if (NIL_P(recv) && BASIC_OP_UNREDEFINED_P(BOP_NIL_P, NIL_REDEFINED_OP_FLAG)) { return Qtrue; } - else if (vm_method_cfunc_is(iseq, cd, recv, rb_false)) { + else if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_false)) { return Qfalse; } else { @@ -7158,9 +7172,9 @@ vm_opt_succ(VALUE recv) } static VALUE -vm_opt_not(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv) +vm_opt_not(struct rb_control_frame_struct *reg_cfp, CALL_DATA cd, VALUE recv) { - if (vm_method_cfunc_is(iseq, cd, recv, rb_obj_not)) { + if (vm_method_cfunc_is(reg_cfp, cd, recv, rb_obj_not)) { return RBOOL(!RTEST(recv)); } else { diff --git a/yjit.c b/yjit.c index 2e7216a1915406..46565cb6c0d035 100644 --- a/yjit.c +++ b/yjit.c @@ -480,7 +480,7 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le // If it's a FINISH frame, just normally exit with a non-Qundef value. cfp->jit_return = leave_exit; } - else if (cfp->jit_return) { + else if (CFP_JIT_RETURN(cfp)) { while (!VM_FRAME_FINISHED_P(cfp)) { if (cfp->jit_return == leave_exit) { // Unlike jit_exec(), leave_exit is not safe on a non-FINISH frame on diff --git a/zjit.h b/zjit.h index f42b77cb356dac..c76a895a87d9d8 100644 --- a/zjit.h +++ b/zjit.h @@ -29,6 +29,10 @@ void rb_zjit_before_ractor_spawn(void); void rb_zjit_tracing_invalidate_all(void); void rb_zjit_invalidate_no_singleton_class(VALUE klass); void rb_zjit_invalidate_root_box(void); +VALUE *rb_zjit_jit_return_pc(void *jit_return); +rb_iseq_t *rb_zjit_jit_return_iseq(void *jit_return); +void rb_zjit_jit_return_set_iseq(void *jit_return, rb_iseq_t *iseq); +bool rb_zjit_jit_return_materialize_block_code(const void *jit_return); #else #define rb_zjit_entry 0 static inline void rb_zjit_compile_iseq(const rb_iseq_t *iseq, bool jit_exception) {} @@ -42,8 +46,72 @@ static inline void rb_zjit_before_ractor_spawn(void) {} static inline void rb_zjit_tracing_invalidate_all(void) {} static inline void rb_zjit_invalidate_no_singleton_class(VALUE klass) {} static inline void rb_zjit_invalidate_root_box(void) {} +static inline VALUE *rb_zjit_jit_return_pc(void *jit_return) { UNREACHABLE_RETURN(0); } +static inline rb_iseq_t *rb_zjit_jit_return_iseq(void *jit_return) { UNREACHABLE_RETURN(0); } +static inline void rb_zjit_jit_return_set_iseq(void *jit_return, rb_iseq_t *iseq) { UNREACHABLE; } +static inline bool rb_zjit_jit_return_materialize_block_code(const void *jit_return) { UNREACHABLE_RETURN(false); } #endif // #if USE_ZJIT #define rb_zjit_enabled_p (rb_zjit_entry != 0) +// Check if cfp->jit_return holds a ZJIT lightweight frame (JITFrame pointer). +// YJIT also uses jit_return (as a return address), so this must only return +// true when ZJIT is enabled and has set jit_return to a JITFrame pointer. +static inline bool +CFP_JIT_RETURN(const rb_control_frame_t *cfp) +{ + if (!rb_zjit_enabled_p) return false; +#if USE_ZJIT + RUBY_ASSERT_ALWAYS(cfp->jit_return != (void *)2); +#endif + return !!cfp->jit_return; +} + +static inline const VALUE* +rb_zjit_cfp_pc(const rb_control_frame_t *cfp) +{ + if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + return rb_zjit_jit_return_pc(cfp->jit_return); + } + else { + return cfp->pc; + } +} + +static inline const rb_iseq_t* +rb_zjit_cfp_iseq(const rb_control_frame_t *cfp) +{ + if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + return rb_zjit_jit_return_iseq(cfp->jit_return); + } + else { + return cfp->iseq; + } +} + +static inline const void* +rb_zjit_cfp_block_code(const rb_control_frame_t *cfp) +{ + if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + return NULL; + } + else { + return cfp->block_code; + } +} + +// Read block_code from a captured block that may live inside a cfp. +// In that case, jit_return is located one word after rb_captured_block. +static inline const void* +rb_zjit_captured_block_code(const struct rb_captured_block *captured) +{ + if (rb_zjit_enabled_p) { + void *jit_return = *(void **)((VALUE *)captured + 3); + if (jit_return) { + return NULL; + } + } + return (const void *)captured->code.val; +} + #endif // #ifndef ZJIT_H diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 3e82efd8f623bc..1541f903411291 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -256,8 +256,7 @@ fn main() { .allowlist_type("iseq_inline_cvar_cache_entry") .blocklist_type("rb_execution_context_.*") // Large struct with various-type fields and an ifdef, so we don't import .opaque_type("rb_execution_context_.*") - .blocklist_type("rb_control_frame_struct") - .opaque_type("rb_control_frame_struct") + .allowlist_type("rb_control_frame_struct") .allowlist_function("rb_vm_bh_to_procval") .allowlist_function("rb_vm_env_write") .allowlist_function("rb_vm_ep_local_ep") diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index fab16166e6262b..650c929fddc84a 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1756,7 +1756,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } }; + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![] } }; asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 03dc02c67846df..3a8f581f69e633 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use std::sync::{Arc, Mutex}; use crate::bitset::BitSet; use crate::codegen::{local_size_and_idx_to_ep_offset, perf_symbol_range_start, perf_symbol_range_end}; -use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; +use crate::cruby::{CfpPtr, IseqPtr, Qundef, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; use crate::hir::{Invariant, SideExitReason}; use crate::hir; use crate::options::{TraceExits, PerfMap, get_option}; @@ -551,6 +551,7 @@ pub struct SideExit { pub pc: Opnd, pub stack: Vec, pub locals: Vec, + pub iseq: IseqPtr, } /// Branch target (something that we can jump to) @@ -2607,16 +2608,13 @@ impl Assembler pub fn compile_exits(&mut self) -> Vec { /// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit. fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) { - let SideExit { pc, stack, locals } = exit; + let SideExit { pc, stack, locals, iseq } = exit; // Side exit blocks are not part of the CFG at the moment, // so we need to manually ensure that patchpoints get padded // so that nobody stomps on us asm.pad_patch_point(); - asm_comment!(asm, "save cfp->pc"); - asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), *pc); - asm_comment!(asm, "save cfp->sp"); asm.lea_into(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); @@ -2633,6 +2631,22 @@ impl Assembler asm.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); } } + + asm_comment!(asm, "save cfp->pc"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), *pc); + + // TODO: can we skip writing this when it's on JIT entry? maybe let materialize_frames handle it? + asm_comment!(asm, "save cfp->iseq"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE::from(*iseq).into()); + + asm_comment!(asm, "save cfp->jit_return"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + + asm_comment!(asm, "materialize caller frames"); + unsafe extern "C" { + fn rb_zjit_materialize_frames(cfp: CfpPtr); + } + asm_ccall!(asm, rb_zjit_materialize_frames, CFP); } /// Tear down the JIT frame and return to the interpreter. diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index 661093e78dc748..4c9911302f0f30 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1392,7 +1392,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![] } }; + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![] } }; asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index a1f7d3f65c714b..7e10ba2fc7c829 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -14,7 +14,7 @@ use crate::invariants::{ track_root_box_assumption }; use crate::gc::append_gc_offsets; -use crate::payload::{get_or_create_iseq_payload, IseqCodePtrs, IseqVersion, IseqVersionRef, IseqStatus}; +use crate::payload::{IseqCodePtrs, IseqStatus, IseqVersion, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; use crate::state::ZJITState; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::{compile_time_ns, exit_compile_error}}; @@ -734,8 +734,8 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { fn gen_objtostring(jit: &mut JITState, asm: &mut Assembler, val: Opnd, cd: *const rb_call_data, state: &FrameState) -> Opnd { gen_prepare_non_leaf_call(jit, asm, state); // TODO: Specialize for immediate types - // Call rb_vm_objtostring(iseq, recv, cd) - let ret = asm_ccall!(asm, rb_vm_objtostring, VALUE::from(jit.iseq).into(), val, Opnd::const_ptr(cd)); + // Call rb_vm_objtostring(cfp, recv, cd) + let ret = asm_ccall!(asm, rb_vm_objtostring, CFP, val, Opnd::const_ptr(cd)); // TODO: Call `to_s` on the receiver if rb_vm_objtostring returns Qundef // Need to replicate what CALL_SIMPLE_METHOD does @@ -1001,8 +1001,8 @@ fn gen_ccall_with_frame( iseq: None, cme, frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, - pc: PC_POISON, specval: block_handler_specval, + write_block_code: false, }); asm_comment!(asm, "switch to new SP register"); @@ -1091,7 +1091,7 @@ fn gen_ccall_variadic( cme, frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, specval: block_handler_specval, - pc: PC_POISON, + write_block_code: false, }); asm_comment!(asm, "switch to new SP register"); @@ -1527,8 +1527,8 @@ fn gen_send_iseq_direct( iseq: Some(iseq), cme, frame_type, - pc: None, specval, + write_block_code: iseq_may_write_block_code(iseq), }); // Write "keyword_bits" to the callee's frame if the callee accepts keywords. @@ -2567,6 +2567,35 @@ fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReaso } } +/// Check if an ISEQ contains instructions that may write to block_code +/// (send, sendforward, invokesuper, invokesuperforward, invokeblock, and their trace variants). +/// These instructions call vm_caller_setup_arg_block which writes to cfp->block_code. +#[allow(non_upper_case_globals)] +fn iseq_may_write_block_code(iseq: IseqPtr) -> bool { + let encoded_size = unsafe { rb_iseq_encoded_size(iseq) }; + let mut insn_idx: u32 = 0; + + while insn_idx < encoded_size { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + let opcode = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } as u32; + + match opcode { + YARVINSN_send | YARVINSN_trace_send | + YARVINSN_sendforward | YARVINSN_trace_sendforward | + YARVINSN_invokesuper | YARVINSN_trace_invokesuper | + YARVINSN_invokesuperforward | YARVINSN_trace_invokesuperforward | + YARVINSN_invokeblock | YARVINSN_trace_invokeblock => { + return true; + } + _ => {} + } + + insn_idx = insn_idx.saturating_add(unsafe { rb_insn_len(VALUE(opcode as usize)) }.try_into().unwrap()); + } + + false +} + /// Save only the PC to CFP. Use this when you need to call gen_save_sp() /// immediately after with a custom stack size (e.g., gen_ccall_with_frame /// adjusts SP to exclude receiver and arguments). @@ -2576,7 +2605,11 @@ fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { gen_incr_counter(asm, Counter::vm_write_pc_count); asm_comment!(asm, "save PC to CFP"); - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(next_pc)); + if let Some(pc) = PC_POISON { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + } + let jit_frame = JITFrame::new(next_pc, state.iseq, !iseq_may_write_block_code(state.iseq)); + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } /// Save the current PC on the CFP as a preparation for calling a C function @@ -2671,7 +2704,9 @@ struct ControlFrame { /// The [`VM_ENV_DATA_INDEX_SPECVAL`] slot of the frame. /// For the type of frames we push, block handler or the parent EP. specval: lir::Opnd, - pc: Option<*const VALUE>, + /// Whether to write block_code = 0 at frame push time. + /// True when the callee ISEQ may write to block_code (has send/invokesuper/invokeblock). + write_block_code: bool, } /// Compile an interpreter frame @@ -2701,25 +2736,33 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C asm_comment!(asm, "push callee control frame"); - if let Some(iseq) = frame.iseq { + if let Some(_iseq) = frame.iseq { // cfp_opnd(RUBY_OFFSET_CFP_PC): written by the callee frame on side-exits, non-leaf calls, or calls with GC // cfp_opnd(RUBY_OFFSET_CFP_SP): written by the callee frame on side-exits, non-leaf calls, or calls with GC - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), VALUE::from(iseq).into()); + //asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), VALUE::from(iseq).into()); + if cfg!(feature = "runtime_checks") { + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), 2.into()); + } + if frame.write_block_code { + asm_comment!(asm, "write block_code for iseq that may use it"); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + } } else { - // C frames don't have a PC and ISEQ in normal operation. - // When runtime checks are enabled we poison the PC so accidental reads stand out. - if let Some(pc) = frame.pc { - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + // C frames don't have a PC and ISEQ in normal operation. ISEQ frames set PC on gen_save_pc_for_gc(). + // When runtime checks are enabled we poison the PC for C frames so accidental reads stand out. + if let (None, Some(pc)) = (frame.iseq, PC_POISON) { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); } let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), 0.into()); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); // just clear a leftover on stack. TODO: make JIT frame for it + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), 0.into()); // TODO: optimize this with JITFrame + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); // TODO: optimize this with JITFrame } asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); let ep = asm.lea(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); } /// Stack overflow check: fails if CFP<=SP at any point in the callee. @@ -2803,6 +2846,7 @@ fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { pc: Opnd::const_ptr(state.pc), stack, locals, + iseq: jit.iseq, } } @@ -2845,19 +2889,33 @@ c_callable! { /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE) -> *const u8 { - with_vm_lock(src_loc!(), || { - // gen_push_frame() doesn't set PC, so we need to set them before exit. - // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. + // Make sure cfp is ready to be scanned by other Ractors and GC before taking the barrier + { + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; let iseq = iseq_call.iseq.get(); let entry_insn_idxs = crate::hir::jit_entry_insns(iseq); + // gen_push_frame() doesn't set PC, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; unsafe { rb_set_cfp_pc(cfp, pc) }; + unsafe { (*cfp).iseq = iseq }; + } + + with_vm_lock(src_loc!(), || { + // TODO: explain why (iseq call's mutability) + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); // Successful JIT-to-JIT calls fill nils to non-parameter locals in generated code. // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, compile_error: &CompileError) { unsafe { + unsafe extern "C" { + fn rb_zjit_materialize_frames(cfp: CfpPtr); + } + rb_zjit_materialize_frames(cfp); + // Set SP which gen_push_frame() doesn't set rb_set_cfp_sp(cfp, sp); @@ -2875,7 +2933,7 @@ c_callable! { } // If we already know we can't compile the ISEQ, fail early without cb.mark_all_executable(). - // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole + // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole // TODO: remove this comment // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. let cb = ZJITState::get_code_block(); let payload = get_or_create_iseq_payload(iseq); diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 5c0b99f182e8d7..e029e5a22800ba 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -165,7 +165,7 @@ unsafe extern "C" { pub fn rb_vm_stack_canary() -> VALUE; pub fn rb_vm_push_cfunc_frame(cme: *const rb_callable_method_entry_t, recv_idx: c_int); pub fn rb_obj_class(klass: VALUE) -> VALUE; - pub fn rb_vm_objtostring(iseq: IseqPtr, recv: VALUE, cd: *const rb_call_data) -> VALUE; + pub fn rb_vm_objtostring(reg_cfp: CfpPtr, recv: VALUE, cd: *const rb_call_data) -> VALUE; } // Renames @@ -373,13 +373,6 @@ pub struct rb_callcache { _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, } -/// Opaque control_frame (CFP) struct from vm_core.h -#[repr(C)] -pub struct rb_control_frame_struct { - _data: [u8; 0], - _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, -} - /// Pointer to a control frame pointer (CFP) pub type CfpPtr = *mut rb_control_frame_struct; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 41ebdb0f55b512..ecabd58c9fdf6f 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1299,6 +1299,16 @@ pub struct rb_block__bindgen_ty_1 { pub proc_: __BindgenUnionField, pub bindgen_union_field: [u64; 3usize], } +#[repr(C)] +pub struct rb_control_frame_struct { + pub pc: *const VALUE, + pub sp: *mut VALUE, + pub iseq: *const rb_iseq_t, + pub self_: VALUE, + pub ep: *const VALUE, + pub block_code: *const ::std::os::raw::c_void, + pub jit_return: *mut ::std::os::raw::c_void, +} pub type rb_control_frame_t = rb_control_frame_struct; #[repr(C)] pub struct rb_proc_t { diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index 40230ccc8db216..0b9de15c866e01 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -90,6 +90,20 @@ pub extern "C" fn rb_zjit_root_update_references() { } let invariants = ZJITState::get_invariants(); invariants.update_references(); + + // Update iseq pointers in all JITFrames for GC compaction. + // rb_execution_context_update only updates JITFrames currently on the stack, + // but JITFrames not on the stack also need their iseq pointers updated + // because the JIT code will reuse them on the next call. + for jit_frame in ZJITState::get_jit_frames().iter_mut() { + let old_iseq = jit_frame.iseq; + if !old_iseq.is_null() { + let new_iseq = unsafe { rb_gc_location(VALUE::from(old_iseq)) }.as_iseq(); + if old_iseq != new_iseq { + jit_frame.iseq = new_iseq; + } + } + } } fn iseq_mark(payload: &IseqPayload) { @@ -208,4 +222,16 @@ fn ranges_overlap(left: &Range, right: &Range) -> bool where T: Partial #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_root_mark() { gc_mark_raw_samples(); + + // Mark iseq pointers in all JITFrames. JITFrames that are currently on the + // stack are also marked via rb_execution_context_mark, but JITFrames not on + // the stack still need their iseqs kept alive because JIT code will reuse them. + if !ZJITState::has_instance() { + return; + } + for jit_frame in ZJITState::get_jit_frames().iter() { + if !jit_frame.iseq.is_null() { + unsafe { rb_gc_mark_movable(VALUE::from(jit_frame.iseq)); } + } + } } diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index d4ac6929345531..7c2052a6ca6ae9 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -6308,7 +6308,7 @@ impl<'a> std::fmt::Display for FunctionGraphvizPrinter<'a> { #[derive(Debug, Clone, PartialEq)] pub struct FrameState { - iseq: IseqPtr, + pub iseq: IseqPtr, insn_idx: usize, // Ruby bytecode instruction pointer pub pc: *const VALUE, diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs new file mode 100644 index 00000000000000..1037cd98d33e7b --- /dev/null +++ b/zjit/src/jit_frame.rs @@ -0,0 +1,184 @@ +use crate::cruby::{IseqPtr, VALUE}; + +// TODO: consider making it C ABI compatible and let C function read it directly +// instead of calling a Rust function +#[derive(Debug)] +pub struct JITFrame { + pub pc: *const VALUE, + pub iseq: IseqPtr, // marked in rb_execution_context_mark + pub materialize_block_code: bool, +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_return_pc(jit_return: *const JITFrame) -> *const VALUE { + unsafe { (*jit_return).pc } +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_return_iseq(jit_return: *const JITFrame) -> IseqPtr { + unsafe { (*jit_return).iseq } +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_return_set_iseq(jit_return: *mut JITFrame, iseq: IseqPtr) { + unsafe { (*jit_return).iseq = iseq; } +} + +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_return_materialize_block_code(jit_return: *const JITFrame) -> bool { + unsafe { (*jit_return).materialize_block_code } +} + +#[cfg(test)] +mod tests { + use crate::cruby::{eval, inspect}; + use insta::assert_snapshot; + + #[test] + fn test_jit_frame_entry_first() { + eval(r#" + def test + itself + callee + end + + def callee + caller + end + + test + "#); + assert_snapshot!(inspect("test.first"), @r#"":4:in 'Object#test'""#); + } + + #[test] + fn test_materialize_one_frame() { + assert_snapshot!(inspect(" + def jit_entry + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + #[test] + fn test_materialize_two_frames() { // materialize caller frames on raise + // At the point of `resuce`, there are two lightweight frames on stack and both need to be + // materialized before passing control to interpreter. + assert_snapshot!(inspect(" + def jit_entry = raise_and_rescue + def raise_and_rescue + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + // TODO: minimize (materialize frames on side exit) + #[test] + fn test_opt_plus_type_guard_nested_exit() { + assert_snapshot!(inspect(" + def side_exit(n) = 1 + n + def jit_frame(n) = 1 + side_exit(n) + def entry(n) = jit_frame(n) + entry(2) # profile send + [entry(2), entry(2.0)] + "), @"[4, 4.0]"); + } + + // TODO: minimize: do not overwrite the top-most frame's PC with jit_frame's PC on invalidation exit + #[test] + fn test_bop_invalidation() { + assert_snapshot!(inspect(r#" + def test + eval("class Integer; def +(_) = 100; end") + 1 + 2 + end + test + test + "#), @"100"); + } + + // TODO: write a test with side exit before writing any jit_return (uninitialized jit_return as of side exit) + + #[test] + fn test_caller_iseq() { + assert_snapshot!(inspect(r#" + def callee = call_caller + def test = callee + + def callee2 = call_caller + def test2 = callee2 + + def call_caller = caller + + test + test2 + test.first + "#), @r#"":2:in 'Object#callee'""#); + } + + #[test] + fn test_iseq_on_raise() { // TODO: minimize + assert_snapshot!(inspect(r#" + def jit_entry(v) = make_range_then_exit(v) + def make_range_then_exit(v) + range = (v..1) + super rescue range + end + jit_entry(0) + jit_entry(0) + jit_entry(0/1r) + "#), @"(0/1)..1"); + } + + #[test] + fn test_iseq_on_raise_on_ensure() { // TODO: minimize + assert_snapshot!(inspect(r#" + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + "#), @r#"["a", "b", "c"]"#); + } + + // TODO: write a test case for GET_ISEQ references in send fallbacks + + // TODO: write a test case for GET_ISEQ references in throw from send fallbacks + + // TODO: write a test case for escaping proc from invokeblock fallback + + // TODO: write a test case for rb_vm_get_sourceline from rb_f_binding + + // TODO: write a test case for svar (iseq reference on rb_vm_svar_lep) +} diff --git a/zjit/src/lib.rs b/zjit/src/lib.rs index 50a48399c2c2d1..1440b6ff6942e6 100644 --- a/zjit/src/lib.rs +++ b/zjit/src/lib.rs @@ -29,6 +29,7 @@ mod profile; mod invariants; mod bitset; mod gc; +mod jit_frame; mod payload; mod json; mod ttycolors; diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs index cb486975bd5d0b..6ee9cc5e778c78 100644 --- a/zjit/src/payload.rs +++ b/zjit/src/payload.rs @@ -4,6 +4,8 @@ use crate::codegen::IseqCallRef; use crate::stats::CompileError; use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; +pub use crate::jit_frame::JITFrame; + /// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. #[derive(Debug)] pub struct IseqPayload { diff --git a/zjit/src/state.rs b/zjit/src/state.rs index b8dcd70358219c..d992dee3f3f4ba 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -1,13 +1,14 @@ //! Runtime state of ZJIT. use crate::codegen::{gen_entry_trampoline, gen_exit_trampoline, gen_exit_trampoline_with_counter, gen_function_stub_hit_trampoline}; -use crate::cruby::{self, rb_bug_panic_hook, rb_vm_insn_count, src_loc, EcPtr, Qnil, Qtrue, rb_vm_insn_addr2opcode, rb_profile_frames, VALUE, VM_INSTRUCTION_SIZE, size_t, rb_gc_mark, with_vm_lock, rust_str_to_id, rb_funcallv, rb_const_get, rb_cRubyVM}; +use crate::cruby::{self, EcPtr, IseqPtr, Qnil, Qtrue, VALUE, VM_INSTRUCTION_SIZE, rb_bug_panic_hook, rb_cRubyVM, rb_const_get, rb_funcallv, rb_gc_mark, rb_profile_frames, rb_vm_insn_addr2opcode, rb_vm_insn_count, rust_str_to_id, size_t, src_loc, with_vm_lock}; use crate::cruby_methods; use cruby::{ID, rb_callable_method_entry, get_def_method_serial, rb_gc_register_mark_object}; use std::sync::atomic::Ordering; use crate::invariants::Invariants; use crate::asm::CodeBlock; use crate::options::{get_option, rb_zjit_prepare_options}; +use crate::payload::JITFrame; use crate::stats::{Counters, InsnCounters, SideExitLocations}; use crate::virtualmem::CodePtr; use std::sync::atomic::AtomicUsize; @@ -70,6 +71,21 @@ pub struct ZJITState { /// Locations of side exists within generated code exit_locations: Option, + + // TODO: consider using raw pointer of JITFrame? + jit_frames: Vec>, +} + +impl JITFrame { + pub fn new(pc: *const VALUE, iseq: IseqPtr, materialize_block_code: bool) -> *const Self { // TODO: move this to jit_frame.rs? + let jit_frame = Box::new(JITFrame { pc, iseq, materialize_block_code }); + let instance = ZJITState::get_instance(); + // FIXME(alan): really, everyone should work with &JITFrame in safe code because &mut exclusivity may not hold + // think about this more + let raw_ptr = jit_frame.as_ref() as *const _; + instance.jit_frames.push(jit_frame); + raw_ptr + } } /// Tracks the initialization progress @@ -147,6 +163,7 @@ impl ZJITState { ccall_counter_pointers: HashMap::new(), iseq_calls_count_pointers: HashMap::new(), exit_locations, + jit_frames: vec![], }; unsafe { ZJIT_STATE = Enabled(zjit_state); } @@ -186,6 +203,10 @@ impl ZJITState { &mut ZJITState::get_instance().invariants } + pub fn get_jit_frames() -> &'static mut Vec> { + &mut ZJITState::get_instance().jit_frames + } + pub fn get_method_annotations() -> &'static cruby_methods::Annotations { &ZJITState::get_instance().method_annotations } From 62c6d036325febe9b7a9e5448d3e2717608700aa Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 23 Mar 2026 13:47:37 -0700 Subject: [PATCH 2/6] Use raw pointers for JITFrame instead of Box --- zjit/src/gc.rs | 14 +++++++------- zjit/src/state.rs | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index 0b9de15c866e01..a7243cef0d38ba 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -3,7 +3,7 @@ use std::ptr::null; use std::{ffi::c_void, ops::Range}; use crate::{cruby::*, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; -use crate::payload::{IseqPayload, IseqVersionRef, get_or_create_iseq_payload}; +use crate::payload::{IseqPayload, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; use crate::stats::Counter::gc_time_ns; use crate::state::gc_mark_raw_samples; @@ -95,12 +95,12 @@ pub extern "C" fn rb_zjit_root_update_references() { // rb_execution_context_update only updates JITFrames currently on the stack, // but JITFrames not on the stack also need their iseq pointers updated // because the JIT code will reuse them on the next call. - for jit_frame in ZJITState::get_jit_frames().iter_mut() { - let old_iseq = jit_frame.iseq; + for &jit_frame in ZJITState::get_jit_frames().iter() { + let old_iseq = unsafe { (*jit_frame).iseq }; if !old_iseq.is_null() { let new_iseq = unsafe { rb_gc_location(VALUE::from(old_iseq)) }.as_iseq(); if old_iseq != new_iseq { - jit_frame.iseq = new_iseq; + unsafe { (*(jit_frame as *mut JITFrame)).iseq = new_iseq; } } } } @@ -229,9 +229,9 @@ pub extern "C" fn rb_zjit_root_mark() { if !ZJITState::has_instance() { return; } - for jit_frame in ZJITState::get_jit_frames().iter() { - if !jit_frame.iseq.is_null() { - unsafe { rb_gc_mark_movable(VALUE::from(jit_frame.iseq)); } + for &jit_frame in ZJITState::get_jit_frames().iter() { + if !unsafe { (*jit_frame).iseq }.is_null() { + unsafe { rb_gc_mark_movable(VALUE::from((*jit_frame).iseq)); } } } } diff --git a/zjit/src/state.rs b/zjit/src/state.rs index d992dee3f3f4ba..2b33c667275534 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -73,7 +73,7 @@ pub struct ZJITState { exit_locations: Option, // TODO: consider using raw pointer of JITFrame? - jit_frames: Vec>, + jit_frames: Vec<*const JITFrame>, } impl JITFrame { @@ -82,8 +82,8 @@ impl JITFrame { let instance = ZJITState::get_instance(); // FIXME(alan): really, everyone should work with &JITFrame in safe code because &mut exclusivity may not hold // think about this more - let raw_ptr = jit_frame.as_ref() as *const _; - instance.jit_frames.push(jit_frame); + let raw_ptr = Box::into_raw(jit_frame) as *const _; + instance.jit_frames.push(raw_ptr); raw_ptr } } @@ -203,7 +203,7 @@ impl ZJITState { &mut ZJITState::get_instance().invariants } - pub fn get_jit_frames() -> &'static mut Vec> { + pub fn get_jit_frames() -> &'static mut Vec<*const JITFrame> { &mut ZJITState::get_instance().jit_frames } From 969294720cd34fe518dc13d32fec0d3675f9d3ea Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 20 Mar 2026 16:13:27 -0700 Subject: [PATCH 3/6] PoC1: Move materialize_frames from side exits to jit_exec/JIT_EXEC Instead of calling rb_zjit_materialize_frames in every side exit's generated code, call zjit_materialize_frames once in jit_exec() and JIT_EXEC() after the ZJIT entry trampoline returns. This reduces generated code size by removing a C function call from each unique side exit. --- vm.c | 11 ++++++++++- vm_exec.h | 1 + zjit/src/backend/lir.rs | 6 ------ zjit/src/codegen.rs | 6 ++---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/vm.c b/vm.c index 323ce0c91584f5..5e015f5b01b6cc 100644 --- a/vm.c +++ b/vm.c @@ -559,6 +559,8 @@ zjit_compile(rb_execution_context_t *ec) # define zjit_compile(ec) ((rb_jit_func_t)0) #endif +static inline void zjit_materialize_frames(rb_control_frame_t *cfp); + // Execute JIT code compiled by yjit_compile() or zjit_compile() static inline VALUE jit_exec(rb_execution_context_t *ec) @@ -578,7 +580,14 @@ jit_exec(rb_execution_context_t *ec) if (zjit_entry) { rb_jit_func_t func = zjit_compile(ec); if (func) { - return ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + VALUE result = ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + // Materialize any remaining lightweight ZJIT frames on side exit. + // This is done here (once per JIT entry) instead of in each side exit + // to reduce generated code size. + if (UNDEF_P(result)) { + zjit_materialize_frames(ec->cfp); + } + return result; } } #endif diff --git a/vm_exec.h b/vm_exec.h index 641ace4eaf29b9..8ec024e204e17d 100644 --- a/vm_exec.h +++ b/vm_exec.h @@ -189,6 +189,7 @@ default: \ rb_jit_func_t func = zjit_compile(ec); \ if (func) { \ val = zjit_entry(ec, ec->cfp, func); \ + if (UNDEF_P(val)) zjit_materialize_frames(ec->cfp); \ } \ } \ } \ diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 3a8f581f69e633..9a792f2f4e56b2 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -2641,12 +2641,6 @@ impl Assembler asm_comment!(asm, "save cfp->jit_return"); asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); - - asm_comment!(asm, "materialize caller frames"); - unsafe extern "C" { - fn rb_zjit_materialize_frames(cfp: CfpPtr); - } - asm_ccall!(asm, rb_zjit_materialize_frames, CFP); } /// Tear down the JIT frame and return to the interpreter. diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 7e10ba2fc7c829..860cd8d099113d 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -2911,10 +2911,8 @@ c_callable! { // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, compile_error: &CompileError) { unsafe { - unsafe extern "C" { - fn rb_zjit_materialize_frames(cfp: CfpPtr); - } - rb_zjit_materialize_frames(cfp); + // Caller frames are materialized by jit_exec() after the entry trampoline returns. + // The current frame's pc and iseq are already set by function_stub_hit before this point. // Set SP which gen_push_frame() doesn't set rb_set_cfp_sp(cfp, sp); From 1ec5c207a15567f853de5ebf0b7a236dfbcdfa61 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 23 Mar 2026 14:17:40 -0700 Subject: [PATCH 4/6] Skip writing jit_return on exits --- vm.c | 1 + vm_exec.h | 5 ++++- zjit/src/backend/lir.rs | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/vm.c b/vm.c index 5e015f5b01b6cc..bed160c8c305d2 100644 --- a/vm.c +++ b/vm.c @@ -585,6 +585,7 @@ jit_exec(rb_execution_context_t *ec) // This is done here (once per JIT entry) instead of in each side exit // to reduce generated code size. if (UNDEF_P(result)) { + ec->cfp->jit_return = 0; zjit_materialize_frames(ec->cfp); } return result; diff --git a/vm_exec.h b/vm_exec.h index 8ec024e204e17d..340fadff517223 100644 --- a/vm_exec.h +++ b/vm_exec.h @@ -189,7 +189,10 @@ default: \ rb_jit_func_t func = zjit_compile(ec); \ if (func) { \ val = zjit_entry(ec, ec->cfp, func); \ - if (UNDEF_P(val)) zjit_materialize_frames(ec->cfp); \ + if (UNDEF_P(val)) { \ + ec->cfp->jit_return = 0; \ + zjit_materialize_frames(ec->cfp); \ + } \ } \ } \ } \ diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 9a792f2f4e56b2..8023b1379993f7 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use std::sync::{Arc, Mutex}; use crate::bitset::BitSet; use crate::codegen::{local_size_and_idx_to_ep_offset, perf_symbol_range_start, perf_symbol_range_end}; -use crate::cruby::{CfpPtr, IseqPtr, Qundef, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; +use crate::cruby::{IseqPtr, Qundef, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; use crate::hir::{Invariant, SideExitReason}; use crate::hir; use crate::options::{TraceExits, PerfMap, get_option}; @@ -2639,8 +2639,8 @@ impl Assembler asm_comment!(asm, "save cfp->iseq"); asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE::from(*iseq).into()); - asm_comment!(asm, "save cfp->jit_return"); - asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + //asm_comment!(asm, "save cfp->jit_return"); + //asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); } /// Tear down the JIT frame and return to the interpreter. From 00c9a4e921d3fd459e62886de5482db47fda7ac7 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 24 Mar 2026 05:49:24 -0700 Subject: [PATCH 5/6] Write JITFrame pointer for C frames, skip iseq/block_code writes Instead of writing 3 fields to C frames (jit_return=0, iseq=0, block_code=0), write a single JITFrame pointer with {pc=NULL, iseq=NULL, materialize_block_code=true}. This eliminates 2 memory writes per C method call. Add rb_zjit_cfp_has_iseq() and rb_zjit_cfp_has_pc() helpers that check JITFrame first when present, since cfp->iseq and cfp->pc may be stale for frames with JITFrame. Replace all cfp->iseq || CFP_JIT_RETURN(cfp) and cfp->pc || CFP_JIT_RETURN(cfp) patterns with the new helpers. --- cont.c | 2 +- gc.c | 2 +- vm.c | 4 ++-- vm_backtrace.c | 14 +++++++------- vm_dump.c | 8 ++++---- zjit.h | 20 ++++++++++++++++++++ zjit/src/codegen.rs | 5 ++--- 7 files changed, 37 insertions(+), 18 deletions(-) diff --git a/cont.c b/cont.c index 1a5f6cb54fb680..ed0af667f3e5dd 100644 --- a/cont.c +++ b/cont.c @@ -1348,7 +1348,7 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if ((cfp->pc || CFP_JIT_RETURN(cfp)) && (cfp->iseq || CFP_JIT_RETURN(cfp))) { + if (rb_zjit_cfp_has_pc(cfp) && rb_zjit_cfp_has_iseq(cfp)) { const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) { callback(iseq, data); diff --git a/gc.c b/gc.c index 2f4dc39675b03a..713f09df683348 100644 --- a/gc.c +++ b/gc.c @@ -1000,7 +1000,7 @@ gc_validate_pc(VALUE obj) rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *cfp = ec->cfp; - if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && (cfp->pc || CFP_JIT_RETURN(cfp))) { + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && rb_zjit_cfp_has_pc(cfp)) { const VALUE *iseq_encoded = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded; const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_size; RUBY_ASSERT(rb_zjit_cfp_pc(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); diff --git a/vm.c b/vm.c index bed160c8c305d2..97d5b80c99ad25 100644 --- a/vm.c +++ b/vm.c @@ -941,7 +941,7 @@ rb_control_frame_t * rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - if (cfp->iseq || CFP_JIT_RETURN(cfp)) { + if (rb_zjit_cfp_has_iseq(cfp)) { return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1997,7 +1997,7 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s VALUE * rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while ((cfp->pc == 0 && !CFP_JIT_RETURN(cfp)) || (cfp->iseq == 0 && !CFP_JIT_RETURN(cfp))) { + while (!rb_zjit_cfp_has_pc(cfp) || !rb_zjit_cfp_has_iseq(cfp)) { if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { struct vm_ifunc *ifunc = (struct vm_ifunc *)rb_zjit_cfp_iseq(cfp); return ifunc->svar_lep; diff --git a/vm_backtrace.c b/vm_backtrace.c index 715443deb824ec..674cfe24274c24 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -102,7 +102,7 @@ calc_node_id(const rb_iseq_t *iseq, const VALUE *pc) int rb_vm_get_sourceline(const rb_control_frame_t *cfp) { - if (VM_FRAME_RUBYFRAME_P(cfp) && (cfp->iseq || CFP_JIT_RETURN(cfp))) { + if (VM_FRAME_RUBYFRAME_P(cfp) && rb_zjit_cfp_has_iseq(cfp)) { const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); int line = calc_lineno(iseq, rb_zjit_cfp_pc(cfp)); if (line != 0) { @@ -688,8 +688,8 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram } for (; cfp != end_cfp && (bt->backtrace_size < num_frames); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (cfp->iseq || CFP_JIT_RETURN(cfp)) { - if (cfp->pc || CFP_JIT_RETURN(cfp)) { + if (rb_zjit_cfp_has_iseq(cfp)) { + if (rb_zjit_cfp_has_pc(cfp)) { if (start_frame > 0) { start_frame--; } @@ -753,7 +753,7 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram // is the one of the caller Ruby frame, so if the last entry is a C frame we find the caller Ruby frame here. if (backpatch_counter > 0) { for (; cfp != end_cfp; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if ((cfp->iseq || CFP_JIT_RETURN(cfp)) && (cfp->pc || CFP_JIT_RETURN(cfp)) && !(skip_internal && is_internal_location(rb_zjit_cfp_iseq(cfp)))) { + if (rb_zjit_cfp_has_iseq(cfp) && rb_zjit_cfp_has_pc(cfp) && !(skip_internal && is_internal_location(rb_zjit_cfp_iseq(cfp)))) { VM_ASSERT(!skip_next_frame); // ISEQ_TYPE_RESCUE/ISEQ_TYPE_ENSURE should have a caller Ruby ISEQ, not a cfunc bt_backpatch_loc(backpatch_counter, loc, rb_zjit_cfp_iseq(cfp), rb_zjit_cfp_pc(cfp)); RB_OBJ_WRITTEN(btobj, Qundef, rb_zjit_cfp_iseq(cfp)); @@ -1020,8 +1020,8 @@ backtrace_each(const rb_execution_context_t *ec, /* SDR(); */ for (i=0, cfp = start_cfp; ivm_stack + ec->vm_stack_size) - cfp); */ - if (cfp->iseq || CFP_JIT_RETURN(cfp)) { - if (cfp->pc || CFP_JIT_RETURN(cfp)) { + if (rb_zjit_cfp_has_iseq(cfp)) { + if (rb_zjit_cfp_has_pc(cfp)) { iter_iseq(arg, cfp); } } @@ -1747,7 +1747,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b end_cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp); for (i=0; ipc != 0 || CFP_JIT_RETURN(cfp))) { + if (VM_FRAME_RUBYFRAME_P_UNCHECKED(cfp) && rb_zjit_cfp_has_pc(cfp)) { if (start > 0) { start--; continue; diff --git a/vm_dump.c b/vm_dump.c index a0398a66ebcc3d..69a24bb79c7fb5 100644 --- a/vm_dump.c +++ b/vm_dump.c @@ -119,7 +119,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c selfstr = ""; } - if (cfp->iseq || CFP_JIT_RETURN(cfp)) { + if (rb_zjit_cfp_has_iseq(cfp)) { iseq = rb_zjit_cfp_iseq(cfp); #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) if (RUBY_VM_IFUNC_P(iseq)) { @@ -132,7 +132,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c line = -1; } else { - if (cfp->pc || CFP_JIT_RETURN(cfp)) { + if (rb_zjit_cfp_has_pc(cfp)) { pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { @@ -341,7 +341,7 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro break; } - if (cfp && (cfp->iseq != 0 || CFP_JIT_RETURN(cfp))) { + if (cfp && rb_zjit_cfp_has_iseq(cfp)) { #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) const rb_iseq_t *resolved_iseq = rb_zjit_cfp_iseq(cfp); if (RUBY_VM_IFUNC_P(resolved_iseq)) { @@ -354,7 +354,7 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro line = -1; } else { - if (cfp->pc || CFP_JIT_RETURN(cfp)) { + if (rb_zjit_cfp_has_pc(cfp)) { iseq = resolved_iseq; pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); diff --git a/zjit.h b/zjit.h index c76a895a87d9d8..f5613d730a0fb6 100644 --- a/zjit.h +++ b/zjit.h @@ -67,6 +67,26 @@ CFP_JIT_RETURN(const rb_control_frame_t *cfp) return !!cfp->jit_return; } +// Returns true if cfp has an ISEQ, either directly or via JITFrame. +// When JITFrame is present, it is authoritative (cfp->iseq may be stale). +// C frames with JITFrame have iseq=NULL, so this returns false for them. +static inline bool +rb_zjit_cfp_has_iseq(const rb_control_frame_t *cfp) +{ + if (CFP_JIT_RETURN(cfp)) return rb_zjit_jit_return_iseq(cfp->jit_return) != NULL; + return !!cfp->iseq; +} + +// Returns true if cfp has a PC, either directly or via JITFrame. +// When JITFrame is present, it is authoritative (cfp->pc may be stale/poisoned). +// C frames with JITFrame have pc=NULL, so this returns false for them. +static inline bool +rb_zjit_cfp_has_pc(const rb_control_frame_t *cfp) +{ + if (CFP_JIT_RETURN(cfp)) return rb_zjit_jit_return_pc(cfp->jit_return) != NULL; + return !!cfp->pc; +} + static inline const VALUE* rb_zjit_cfp_pc(const rb_control_frame_t *cfp) { diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 860cd8d099113d..0d56c8c7357944 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -2755,9 +2755,8 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C } let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); // just clear a leftover on stack. TODO: make JIT frame for it - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), 0.into()); // TODO: optimize this with JITFrame - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); // TODO: optimize this with JITFrame + let jit_frame = JITFrame::new(std::ptr::null(), std::ptr::null(), true); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); From bf4d608e6193b44094a3928213d3ceb73d23bf6d Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 24 Mar 2026 06:36:59 -0700 Subject: [PATCH 6/6] Skip GC relocation of stale block_code on C frames with JITFrame When CFP_JIT_RETURN is true for a C frame (iseq is NULL in JITFrame), block_code in the CFP is stale and must not be passed to rb_gc_location during GC compaction. Only relocate iseq and block_code for ISEQ frames with JITFrame. --- vm.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/vm.c b/vm.c index 97d5b80c99ad25..cf6de2fcb9fe6a 100644 --- a/vm.c +++ b/vm.c @@ -3671,11 +3671,16 @@ rb_execution_context_update(rb_execution_context_t *ec) const VALUE *ep = cfp->ep; cfp->self = rb_gc_location(cfp->self); if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { - rb_zjit_jit_return_set_iseq(cfp->jit_return, (rb_iseq_t *)rb_gc_location((VALUE)rb_zjit_cfp_iseq(cfp))); - // block_code may have been written by the JIT caller (gen_block_handler_specval) - // for passing blocks. It must be relocated even when jit_return is set, because - // Proc creation (block_setup) copies it from the CFP's captured block. - cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + const rb_iseq_t *iseq = rb_zjit_jit_return_iseq(cfp->jit_return); + if (iseq) { + // ISEQ frame with JITFrame: relocate iseq in JITFrame and block_code in CFP + rb_zjit_jit_return_set_iseq(cfp->jit_return, (rb_iseq_t *)rb_gc_location((VALUE)iseq)); + // block_code may have been written by the JIT caller (gen_block_handler_specval) + // for passing blocks. It must be relocated even when jit_return is set, because + // Proc creation (block_setup) copies it from the CFP's captured block. + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } + // C frame with JITFrame: iseq is NULL, block_code is stale — skip relocation } else { cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq);